Skip to content

Loader Class

Loader classes.

SQLLoader

Class for loading and parsing SQL queries.

Source code in amee_utils/loader.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
class SQLLoader:
    """Class for loading and parsing SQL queries."""

    def __init__(self, spark: SparkSession) -> None:
        """Initialise the SQLLoader.

        Parameters
        ----------
        spark : SparkSession
            The SparkSession object to use for executing queries.

        Returns
        -------
        None
        """
        self.spark = spark

    @staticmethod
    def _read_file(path: Path) -> str:
        """Read the contents of a file and return it as a string.

        Parameters
        ----------
        path : Path
            The path to the file to read.

        Returns
        -------
        str
            The contents of the file as a string.

        Raises
        ------
        FileNotFoundError
            If the specified SQL file is not found.
        """
        try:
            with path.open("r") as f:
                return f.read()

        except FileNotFoundError as e:
            raise FileNotFoundError(f"SQL file not found: {path}. Please check the path and file name.") from e

    def parse(self, query: str) -> DataFrame:
        """Parse and execute an SQL query using the SparkSession.

        Drop duplicates, and return the result as a PySpark DataFrame.

        Parameters
        ----------
        query : str
            The SQL query to parse and execute.

        Returns
        -------
        DataFrame
            The result of the SQL query execution as a PySpark DataFrame.
        """
        return self.spark.sql(sqlQuery=query)

    def from_file(self, path: Path) -> DataFrame:
        """Load an SQL query from a file, parse it, and execute it using the SparkSession.

        Returns the parsed query result as a PySpark DataFrame.

        Parameters
        ----------
        path : Path
            The path to the file containing the SQL query.

        Returns
        -------
        DataFrame
            The result of the SQL query execution as a PySpark DataFrame.
        """
        query = self._read_file(path=path)

        return self.parse(query)

from_file(path)

Load an SQL query from a file, parse it, and execute it using the SparkSession.

Returns the parsed query result as a PySpark DataFrame.

Parameters:

Name Type Description Default
path Path

The path to the file containing the SQL query.

 required

Returns:

Type Description
DataFrame

The result of the SQL query execution as a PySpark DataFrame.
Source code in amee_utils/loader.py 
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def from_file(self, path: Path) -> DataFrame:
    """Load an SQL query from a file, parse it, and execute it using the SparkSession.

    Returns the parsed query result as a PySpark DataFrame.

    Parameters
    ----------
    path : Path
        The path to the file containing the SQL query.

    Returns
    -------
    DataFrame
        The result of the SQL query execution as a PySpark DataFrame.
    """
    query = self._read_file(path=path)

    return self.parse(query)

parse(query)

Parse and execute an SQL query using the SparkSession.

Drop duplicates, and return the result as a PySpark DataFrame.

Parameters:

Name Type Description Default
query str

The SQL query to parse and execute.

 required

Returns:

Type Description
DataFrame

The result of the SQL query execution as a PySpark DataFrame.
Source code in amee_utils/loader.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def parse(self, query: str) -> DataFrame:
    """Parse and execute an SQL query using the SparkSession.

    Drop duplicates, and return the result as a PySpark DataFrame.

    Parameters
    ----------
    query : str
        The SQL query to parse and execute.

    Returns
    -------
    DataFrame
        The result of the SQL query execution as a PySpark DataFrame.
    """
    return self.spark.sql(sqlQuery=query)