Version: 1.4.4

SparkAzureBlobStorageDatasource

Signature

class great_expectations.datasource.fluent.SparkAzureBlobStorageDatasource(
 *,
 type: Literal['spark_abs'] = 'spark_abs',
 name: str,
 id: Optional[uuid.UUID] = None,
 assets: List[Union[great_expectations.datasource.fluent.data_asset.path.spark.csv_asset.CSVAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.csv_asset.DirectoryCSVAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.parquet_asset.ParquetAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.parquet_asset.DirectoryParquetAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.orc_asset.ORCAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.orc_asset.DirectoryORCAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.json_asset.JSONAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.json_asset.DirectoryJSONAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.text_asset.TextAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.text_asset.DirectoryTextAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.delta_asset.DeltaAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.delta_asset.DirectoryDeltaAsset]] = [],
 spark_config: Optional[Dict[pydantic.v1.types.StrictStr,
 Union[pydantic.v1.types.StrictStr,
 pydantic.v1.types.StrictInt,
 pydantic.v1.types.StrictFloat,
 pydantic.v1.types.StrictBool]]] = None,
 force_reuse_spark_context: bool = True,
 persist: bool = True,
 azure_options: Dict[str,
 Union[great_expectations.datasource.fluent.config_str.ConfigStr,
 Any]] = {}
)

SparkAzureBlobStorageDatasource is a subclass of SparkDatasource which connects to Azure Blob Storage.

Methods

add_csv_asset

Signature

add_csv_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd83190ae40> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd83190af00> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd83190b050> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd83190b200> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd83190b2c0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 sep: typing.Optional[str] = None,
 encoding: typing.Optional[str] = None,
 quote: typing.Optional[str] = None,
 escape: typing.Optional[str] = None,
 comment: typing.Optional[str] = None,
 header: typing.Optional[typing.Union[bool,
 str]] = None,
 inferSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreLeadingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreTrailingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 nullValue: typing.Optional[str] = None,
 nanValue: typing.Optional[str] = None,
 positiveInf: typing.Optional[str] = None,
 negativeInf: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 maxColumns: typing.Optional[typing.Union[int,
 str]] = None,
 maxCharsPerColumn: typing.Optional[typing.Union[int,
 str]] = None,
 maxMalformedLogPerPartition: typing.Optional[typing.Union[int,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 charToEscapeQuoteEscaping: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 enforceSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 emptyValue: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 lineSep: typing.Optional[str] = None,
 unescapedQuoteHandling: typing.Optional[typing.Literal['STOP_AT_CLOSING_QUOTE',
 'BACK_TO_DELIMITER',
 'STOP_AT_DELIMITER',
 'SKIP_VALUE',
 'RAISE_ERROR']] = None
) → pydantic.BaseModel

Add a csv asset to the datasource.

add_delta_asset

Signature

add_delta_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd83176ad50> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd83176ae10> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd83176af60> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd83176b110> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd83176b1d0> = None,
 timestampAsOf: typing.Optional[str] = None,
 versionAsOf: typing.Optional[str] = None
) → pydantic.BaseModel

Add a delta asset to the datasource.

add_directory_csv_asset

Signature

add_directory_csv_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd831769550> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd831769610> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd831769760> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd831769910> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317699d0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 sep: typing.Optional[str] = None,
 encoding: typing.Optional[str] = None,
 quote: typing.Optional[str] = None,
 escape: typing.Optional[str] = None,
 comment: typing.Optional[str] = None,
 header: typing.Optional[typing.Union[bool,
 str]] = None,
 inferSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreLeadingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreTrailingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 nullValue: typing.Optional[str] = None,
 nanValue: typing.Optional[str] = None,
 positiveInf: typing.Optional[str] = None,
 negativeInf: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 maxColumns: typing.Optional[typing.Union[int,
 str]] = None,
 maxCharsPerColumn: typing.Optional[typing.Union[int,
 str]] = None,
 maxMalformedLogPerPartition: typing.Optional[typing.Union[int,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 charToEscapeQuoteEscaping: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 enforceSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 emptyValue: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 lineSep: typing.Optional[str] = None,
 unescapedQuoteHandling: typing.Optional[typing.Literal['STOP_AT_CLOSING_QUOTE',
 'BACK_TO_DELIMITER',
 'STOP_AT_DELIMITER',
 'SKIP_VALUE',
 'RAISE_ERROR']] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_csv asset to the datasource.

add_directory_delta_asset

Signature

add_directory_delta_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd83176bfe0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317840e0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd831784230> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317843e0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317844a0> = None,
 timestampAsOf: typing.Optional[str] = None,
 versionAsOf: typing.Optional[str] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_delta asset to the datasource.

add_directory_json_asset

Signature

add_directory_json_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317b9ee0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317b9fa0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd8317ba0f0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317ba2a0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317ba360> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 primitivesAsString: typing.Optional[typing.Union[bool,
 str]] = None,
 prefersDecimal: typing.Optional[typing.Union[bool,
 str]] = None,
 allowComments: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedFieldNames: typing.Optional[typing.Union[bool,
 str]] = None,
 allowSingleQuotes: typing.Optional[typing.Union[bool,
 str]] = None,
 allowNumericLeadingZero: typing.Optional[typing.Union[bool,
 str]] = None,
 allowBackslashEscapingAnyCharacter: typing.Optional[typing.Union[bool,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedControlChars: typing.Optional[typing.Union[bool,
 str]] = None,
 lineSep: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 dropFieldIfAllNull: typing.Optional[typing.Union[bool,
 str]] = None,
 encoding: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 allowNonNumericNumbers: typing.Optional[typing.Union[bool,
 str]] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_json asset to the datasource.

add_directory_orc_asset

Signature

add_directory_orc_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317d9760> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317d9820> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd8317d9970> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317d9b20> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317d9be0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = False,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_orc asset to the datasource.

add_directory_parquet_asset

Signature

add_directory_parquet_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317fd2b0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317fd370> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd8317fd4c0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317fd670> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317fd730> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 datetimeRebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 int96RebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_parquet asset to the datasource.

add_directory_text_asset

Signature

add_directory_text_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd831818b30> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd831818bf0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd831818d40> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd831818ef0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd831818fb0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 wholetext: bool = False,
 lineSep: typing.Optional[str] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_text asset to the datasource.

add_json_asset

Signature

add_json_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317878c0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd831787a10> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd831787b60> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd831787d10> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd831787dd0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 primitivesAsString: typing.Optional[typing.Union[bool,
 str]] = None,
 prefersDecimal: typing.Optional[typing.Union[bool,
 str]] = None,
 allowComments: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedFieldNames: typing.Optional[typing.Union[bool,
 str]] = None,
 allowSingleQuotes: typing.Optional[typing.Union[bool,
 str]] = None,
 allowNumericLeadingZero: typing.Optional[typing.Union[bool,
 str]] = None,
 allowBackslashEscapingAnyCharacter: typing.Optional[typing.Union[bool,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedControlChars: typing.Optional[typing.Union[bool,
 str]] = None,
 lineSep: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 dropFieldIfAllNull: typing.Optional[typing.Union[bool,
 str]] = None,
 encoding: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 allowNonNumericNumbers: typing.Optional[typing.Union[bool,
 str]] = None
) → pydantic.BaseModel

Add a json asset to the datasource.

add_orc_asset

Signature

add_orc_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317d8290> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317d8350> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd8317d84a0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317d8650> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317d8710> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = False
) → pydantic.BaseModel

Add an orc asset to the datasource.

add_parquet_asset

Signature

add_parquet_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317dbc80> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317dbd40> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd8317dbe90> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317fc080> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317fc140> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 datetimeRebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 int96RebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None
) → pydantic.BaseModel

Add a parquet asset to the datasource.

add_text_asset

Signature

add_text_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7fd8317ff5c0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7fd8317ff680> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fd8317ff7d0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fd8317ff980> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7fd8317ffa40> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 wholetext: bool = False,
 lineSep: typing.Optional[str] = None
) → pydantic.BaseModel

Add a text asset to the datasource.

delete_asset

Signature

delete_asset(
 name: str
) → None

Removes the DataAsset referred to by asset_name from internal list of available DataAsset objects.

Parameters

Name	Description
name	name of DataAsset to be deleted.

get_asset

Signature

get_asset(
 name: str
) → great_expectations.datasource.fluent.interfaces._DataAssetT

Returns the DataAsset referred to by asset_name

Parameters

Name	Description
name	name of DataAsset sought.

Returns

Type	Description
great_expectations.datasource.fluent.interfaces._DataAssetT	if named "DataAsset" object exists; otherwise, exception is raised.