Skip to main content
Version: 1.23.2

SparkFilesystemDatasource

Signature

class great_expectations.datasource.fluent.SparkFilesystemDatasource(
 *,
 type: Literal['spark_filesystem'] = 'spark_filesystem',
 name: str,
 id: Optional[uuid.UUID] = None,
 assets: List[Union[great_expectations.datasource.fluent.data_asset.path.spark.csv_asset.CSVAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.csv_asset.DirectoryCSVAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.parquet_asset.ParquetAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.parquet_asset.DirectoryParquetAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.orc_asset.ORCAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.orc_asset.DirectoryORCAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.json_asset.JSONAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.json_asset.DirectoryJSONAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.text_asset.TextAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.text_asset.DirectoryTextAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.delta_asset.DeltaAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.delta_asset.DirectoryDeltaAsset]] = [],
 spark_config: Optional[Dict[pydantic.v1.types.StrictStr,
 Union[pydantic.v1.types.StrictStr,
 pydantic.v1.types.StrictInt,
 pydantic.v1.types.StrictFloat,
 pydantic.v1.types.StrictBool]]] = None,
 force_reuse_spark_context: bool = True,
 persist: bool = True,
 base_directory: pathlib.Path,
 data_context_root_directory: Optional[pathlib.Path] = None
)

SparkFilesystemDatasource is a subclass of SparkDatasource which connects to the filesystem.

Methods

add_csv_asset

Signature

add_csv_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc522726f0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc522727b0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc52272900> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc52272ab0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc52272b70> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 sep: typing.Optional[str] = None,
 encoding: typing.Optional[str] = None,
 quote: typing.Optional[str] = None,
 escape: typing.Optional[str] = None,
 comment: typing.Optional[str] = None,
 header: typing.Optional[typing.Union[bool,
 str]] = None,
 inferSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreLeadingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreTrailingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 nullValue: typing.Optional[str] = None,
 nanValue: typing.Optional[str] = None,
 positiveInf: typing.Optional[str] = None,
 negativeInf: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 maxColumns: typing.Optional[typing.Union[int,
 str]] = None,
 maxCharsPerColumn: typing.Optional[typing.Union[int,
 str]] = None,
 maxMalformedLogPerPartition: typing.Optional[typing.Union[int,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 charToEscapeQuoteEscaping: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 enforceSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 emptyValue: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 lineSep: typing.Optional[str] = None,
 unescapedQuoteHandling: typing.Optional[typing.Literal['STOP_AT_CLOSING_QUOTE',
 'BACK_TO_DELIMITER',
 'STOP_AT_DELIMITER',
 'SKIP_VALUE',
 'RAISE_ERROR']] = None
) → pydantic.BaseModel

Add a csv asset to the datasource.

add_delta_asset

Signature

add_delta_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc522d6630> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc522d66f0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc522d6840> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc522d69f0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc522d6ab0> = None,
 timestampAsOf: typing.Optional[str] = None,
 versionAsOf: typing.Optional[str] = None
) → pydantic.BaseModel

Add a delta asset to the datasource.

add_directory_csv_asset

Signature

add_directory_csv_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc522d4e60> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc522d4f20> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc522d5070> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc522d5220> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc522d52e0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 sep: typing.Optional[str] = None,
 encoding: typing.Optional[str] = None,
 quote: typing.Optional[str] = None,
 escape: typing.Optional[str] = None,
 comment: typing.Optional[str] = None,
 header: typing.Optional[typing.Union[bool,
 str]] = None,
 inferSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreLeadingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreTrailingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 nullValue: typing.Optional[str] = None,
 nanValue: typing.Optional[str] = None,
 positiveInf: typing.Optional[str] = None,
 negativeInf: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 maxColumns: typing.Optional[typing.Union[int,
 str]] = None,
 maxCharsPerColumn: typing.Optional[typing.Union[int,
 str]] = None,
 maxMalformedLogPerPartition: typing.Optional[typing.Union[int,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 charToEscapeQuoteEscaping: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 enforceSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 emptyValue: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 lineSep: typing.Optional[str] = None,
 unescapedQuoteHandling: typing.Optional[typing.Literal['STOP_AT_CLOSING_QUOTE',
 'BACK_TO_DELIMITER',
 'STOP_AT_DELIMITER',
 'SKIP_VALUE',
 'RAISE_ERROR']] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_csv asset to the datasource.

add_directory_delta_asset

Signature

add_directory_delta_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc522d78c0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc522d7980> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc522d7ad0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc522d7c80> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc522d7d40> = None,
 timestampAsOf: typing.Optional[str] = None,
 versionAsOf: typing.Optional[str] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_delta asset to the datasource.

add_directory_json_asset

Signature

add_directory_json_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc5211d760> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc5211d820> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc5211d970> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc5211db20> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc5211dbe0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 primitivesAsString: typing.Optional[typing.Union[bool,
 str]] = None,
 prefersDecimal: typing.Optional[typing.Union[bool,
 str]] = None,
 allowComments: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedFieldNames: typing.Optional[typing.Union[bool,
 str]] = None,
 allowSingleQuotes: typing.Optional[typing.Union[bool,
 str]] = None,
 allowNumericLeadingZero: typing.Optional[typing.Union[bool,
 str]] = None,
 allowBackslashEscapingAnyCharacter: typing.Optional[typing.Union[bool,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedControlChars: typing.Optional[typing.Union[bool,
 str]] = None,
 lineSep: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 dropFieldIfAllNull: typing.Optional[typing.Union[bool,
 str]] = None,
 encoding: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 allowNonNumericNumbers: typing.Optional[typing.Union[bool,
 str]] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_json asset to the datasource.

add_directory_orc_asset

Signature

add_directory_orc_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc52138f50> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc52139010> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc52139160> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc52139310> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc521393d0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = False,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_orc asset to the datasource.

add_directory_parquet_asset

Signature

add_directory_parquet_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc52164aa0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc52164b60> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc52164cb0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc52164e60> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc52164f20> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 datetimeRebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 int96RebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_parquet asset to the datasource.

add_directory_text_asset

Signature

add_directory_text_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc521782c0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc52178380> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc521784d0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc52178680> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc52178740> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 wholetext: bool = False,
 lineSep: typing.Optional[str] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_text asset to the datasource.

add_json_asset

Signature

add_json_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc520f3170> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc520f32c0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc520f3410> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc520f35c0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc520f3680> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 primitivesAsString: typing.Optional[typing.Union[bool,
 str]] = None,
 prefersDecimal: typing.Optional[typing.Union[bool,
 str]] = None,
 allowComments: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedFieldNames: typing.Optional[typing.Union[bool,
 str]] = None,
 allowSingleQuotes: typing.Optional[typing.Union[bool,
 str]] = None,
 allowNumericLeadingZero: typing.Optional[typing.Union[bool,
 str]] = None,
 allowBackslashEscapingAnyCharacter: typing.Optional[typing.Union[bool,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedControlChars: typing.Optional[typing.Union[bool,
 str]] = None,
 lineSep: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 dropFieldIfAllNull: typing.Optional[typing.Union[bool,
 str]] = None,
 encoding: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 allowNonNumericNumbers: typing.Optional[typing.Union[bool,
 str]] = None
) → pydantic.BaseModel

Add a json asset to the datasource.

add_orc_asset

Signature

add_orc_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc5211fa10> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc5211fad0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc5211fc20> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc5211fdd0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc5211fe90> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = False
) → pydantic.BaseModel

Add an orc asset to the datasource.

add_parquet_asset

Signature

add_parquet_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc5213b470> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc5213b530> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc5213b680> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc5213b830> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc5213b8f0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 datetimeRebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 int96RebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None
) → pydantic.BaseModel

Add a parquet asset to the datasource.

add_text_asset

Signature

add_text_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fbc52166d20> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fbc52166de0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fbc52166f30> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fbc521670e0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fbc521671a0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 wholetext: bool = False,
 lineSep: typing.Optional[str] = None
) → pydantic.BaseModel

Add a text asset to the datasource.

delete_asset

Signature

delete_asset(
 name: str
) → None

Removes the DataAsset referred to by asset_name from internal list of available DataAsset objects.

Parameters

NameDescription

name

name of DataAsset to be deleted.

get_asset

Signature

get_asset(
 name: str
) → great_expectations.datasource.fluent.interfaces._DataAssetT

Returns the DataAsset referred to by asset_name

Parameters

NameDescription

name

name of DataAsset sought.

Returns

TypeDescription

great_expectations.datasource.fluent.interfaces._DataAssetT

if named "DataAsset" object exists; otherwise, exception is raised.