SparkGoogleCloudStorageDatasource

Signature

class great_expectations.datasource.fluent.SparkGoogleCloudStorageDatasource(
 *,
 type: Literal['spark_gcs'] = 'spark_gcs',
 name: str,
 id: Optional[uuid.UUID] = None,
 assets: List[Union[great_expectations.datasource.fluent.data_asset.path.spark.csv_asset.CSVAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.csv_asset.DirectoryCSVAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.parquet_asset.ParquetAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.parquet_asset.DirectoryParquetAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.orc_asset.ORCAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.orc_asset.DirectoryORCAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.json_asset.JSONAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.json_asset.DirectoryJSONAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.text_asset.TextAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.text_asset.DirectoryTextAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.delta_asset.DeltaAsset,
 great_expectations.datasource.fluent.data_asset.path.spark.delta_asset.DirectoryDeltaAsset]] = [],
 spark_config: Optional[Dict[pydantic.v1.types.StrictStr,
 Union[pydantic.v1.types.StrictStr,
 pydantic.v1.types.StrictInt,
 pydantic.v1.types.StrictFloat,
 pydantic.v1.types.StrictBool]]] = None,
 force_reuse_spark_context: bool = True,
 persist: bool = True,
 bucket_or_name: str,
 gcs_options: Dict[str,
 Union[great_expectations.datasource.fluent.config_str.ConfigStr,
 Any]] = {}
)

SparkGoogleCloudStorageDatasource is a subclass of SparkDatasource which connects to Google Cloud Storage.

Methods

add_csv_asset

Signature

add_csv_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3e629c0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3e62a80> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3e62bd0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3e62d80> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3e62e40> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 sep: typing.Optional[str] = None,
 encoding: typing.Optional[str] = None,
 quote: typing.Optional[str] = None,
 escape: typing.Optional[str] = None,
 comment: typing.Optional[str] = None,
 header: typing.Optional[typing.Union[bool,
 str]] = None,
 inferSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreLeadingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreTrailingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 nullValue: typing.Optional[str] = None,
 nanValue: typing.Optional[str] = None,
 positiveInf: typing.Optional[str] = None,
 negativeInf: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 maxColumns: typing.Optional[typing.Union[int,
 str]] = None,
 maxCharsPerColumn: typing.Optional[typing.Union[int,
 str]] = None,
 maxMalformedLogPerPartition: typing.Optional[typing.Union[int,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 charToEscapeQuoteEscaping: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 enforceSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 emptyValue: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 lineSep: typing.Optional[str] = None,
 unescapedQuoteHandling: typing.Optional[typing.Literal['STOP_AT_CLOSING_QUOTE',
 'BACK_TO_DELIMITER',
 'STOP_AT_DELIMITER',
 'SKIP_VALUE',
 'RAISE_ERROR']] = None
) → pydantic.BaseModel

Add a csv asset to the datasource.

add_delta_asset

Signature

add_delta_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0a900> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0a9c0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0ab10> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0acc0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0ad80> = None,
 timestampAsOf: typing.Optional[str] = None,
 versionAsOf: typing.Optional[str] = None
) → pydantic.BaseModel

Add a delta asset to the datasource.

add_directory_csv_asset

Signature

add_directory_csv_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3d09130> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3d091f0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3d09340> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3d094f0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3d095b0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 sep: typing.Optional[str] = None,
 encoding: typing.Optional[str] = None,
 quote: typing.Optional[str] = None,
 escape: typing.Optional[str] = None,
 comment: typing.Optional[str] = None,
 header: typing.Optional[typing.Union[bool,
 str]] = None,
 inferSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreLeadingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 ignoreTrailingWhiteSpace: typing.Optional[typing.Union[bool,
 str]] = None,
 nullValue: typing.Optional[str] = None,
 nanValue: typing.Optional[str] = None,
 positiveInf: typing.Optional[str] = None,
 negativeInf: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 maxColumns: typing.Optional[typing.Union[int,
 str]] = None,
 maxCharsPerColumn: typing.Optional[typing.Union[int,
 str]] = None,
 maxMalformedLogPerPartition: typing.Optional[typing.Union[int,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 charToEscapeQuoteEscaping: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 enforceSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 emptyValue: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 lineSep: typing.Optional[str] = None,
 unescapedQuoteHandling: typing.Optional[typing.Literal['STOP_AT_CLOSING_QUOTE',
 'BACK_TO_DELIMITER',
 'STOP_AT_DELIMITER',
 'SKIP_VALUE',
 'RAISE_ERROR']] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_csv asset to the datasource.

add_directory_delta_asset

Signature

add_directory_delta_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0bb90> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0bc50> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0bda0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3d0bf50> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3d2c050> = None,
 timestampAsOf: typing.Optional[str] = None,
 versionAsOf: typing.Optional[str] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_delta asset to the datasource.

add_directory_json_asset

Signature

add_directory_json_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3d51ac0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3d51b80> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3d51cd0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3d51e80> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3d51f40> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 primitivesAsString: typing.Optional[typing.Union[bool,
 str]] = None,
 prefersDecimal: typing.Optional[typing.Union[bool,
 str]] = None,
 allowComments: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedFieldNames: typing.Optional[typing.Union[bool,
 str]] = None,
 allowSingleQuotes: typing.Optional[typing.Union[bool,
 str]] = None,
 allowNumericLeadingZero: typing.Optional[typing.Union[bool,
 str]] = None,
 allowBackslashEscapingAnyCharacter: typing.Optional[typing.Union[bool,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedControlChars: typing.Optional[typing.Union[bool,
 str]] = None,
 lineSep: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 dropFieldIfAllNull: typing.Optional[typing.Union[bool,
 str]] = None,
 encoding: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 allowNonNumericNumbers: typing.Optional[typing.Union[bool,
 str]] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_json asset to the datasource.

add_directory_orc_asset

Signature

add_directory_orc_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3b79370> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3b79430> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3b79580> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3b79730> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3b797f0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = False,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_orc asset to the datasource.

add_directory_parquet_asset

Signature

add_directory_parquet_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9cec0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9cf80> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9d0d0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9d280> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9d340> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 datetimeRebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 int96RebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_parquet asset to the datasource.

add_directory_text_asset

Signature

add_directory_text_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3bb86b0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3bb8770> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3bb88c0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3bb8a70> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3bb8b30> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 wholetext: bool = False,
 lineSep: typing.Optional[str] = None,
 data_directory: pathlib.Path
) → pydantic.BaseModel

Add a directory_text asset to the datasource.

add_json_asset

Signature

add_json_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3d2f3b0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3d2f590> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3d2f740> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3d2f8f0> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3d2f9b0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 schema: typing.Optional[typing.Union[great_expectations.datasource.fluent.serializable_types.pyspark.SerializableStructType,
 str]] = None,
 primitivesAsString: typing.Optional[typing.Union[bool,
 str]] = None,
 prefersDecimal: typing.Optional[typing.Union[bool,
 str]] = None,
 allowComments: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedFieldNames: typing.Optional[typing.Union[bool,
 str]] = None,
 allowSingleQuotes: typing.Optional[typing.Union[bool,
 str]] = None,
 allowNumericLeadingZero: typing.Optional[typing.Union[bool,
 str]] = None,
 allowBackslashEscapingAnyCharacter: typing.Optional[typing.Union[bool,
 str]] = None,
 mode: typing.Optional[typing.Literal['PERMISSIVE',
 'DROPMALFORMED',
 'FAILFAST']] = None,
 columnNameOfCorruptRecord: typing.Optional[str] = None,
 dateFormat: typing.Optional[str] = None,
 timestampFormat: typing.Optional[str] = None,
 multiLine: typing.Optional[typing.Union[bool,
 str]] = None,
 allowUnquotedControlChars: typing.Optional[typing.Union[bool,
 str]] = None,
 lineSep: typing.Optional[str] = None,
 samplingRatio: typing.Optional[typing.Union[float,
 str]] = None,
 dropFieldIfAllNull: typing.Optional[typing.Union[bool,
 str]] = None,
 encoding: typing.Optional[str] = None,
 locale: typing.Optional[str] = None,
 allowNonNumericNumbers: typing.Optional[typing.Union[bool,
 str]] = None
) → pydantic.BaseModel

Add a json asset to the datasource.

add_orc_asset

Signature

add_orc_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3d53e30> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3d53ef0> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3b78080> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3b78230> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3b782f0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = False
) → pydantic.BaseModel

Add an orc asset to the datasource.

add_parquet_asset

Signature

add_parquet_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3b7b8c0> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3b7b980> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3b7bad0> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3b7bc80> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3b7bd40> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 mergeSchema: typing.Optional[typing.Union[bool,
 str]] = None,
 datetimeRebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None,
 int96RebaseMode: typing.Optional[typing.Literal['EXCEPTION',
 'CORRECTED',
 'LEGACY']] = None
) → pydantic.BaseModel

Add a parquet asset to the datasource.

add_text_asset

Signature

add_text_asset(
 name: str,
 *,
 id: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9f170> = None,
 order_by: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9f230> = None,
 batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9f380> = None,
 batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9f530> = None,
 connect_options: <pydantic.v1.fields.DeferredType object at 0x7f11a3b9f5f0> = None,
 pathGlobFilter: typing.Optional[typing.Union[bool,
 str]] = None,
 recursiveFileLookup: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedBefore: typing.Optional[typing.Union[bool,
 str]] = None,
 modifiedAfter: typing.Optional[typing.Union[bool,
 str]] = None,
 wholetext: bool = False,
 lineSep: typing.Optional[str] = None
) → pydantic.BaseModel

Add a text asset to the datasource.

delete_asset

Signature

delete_asset(
 name: str
) → None

Removes the DataAsset referred to by asset_name from internal list of available DataAsset objects.

Parameters

Name	Description
name	name of DataAsset to be deleted.

get_asset

Signature

get_asset(
 name: str
) → great_expectations.datasource.fluent.interfaces._DataAssetT

Returns the DataAsset referred to by asset_name

Parameters

Name	Description
name	name of DataAsset sought.

Returns

Type	Description
great_expectations.datasource.fluent.interfaces._DataAssetT	if named "DataAsset" object exists; otherwise, exception is raised.