PandasFilesystemDatasource
Pandas based Datasource for filesystem based data assets.
Add a csv asset to the datasource.
Add an excel asset to the datasource.
Add a feather asset to the datasource.
Add a fwf asset to the datasource.
Add a hdf asset to the datasource.
Add a html asset to the datasource.
Add an iceberg asset to the datasource.
Add a json asset to the datasource.
Add an orc asset to the datasource.
Add a parquet asset to the datasource.
Add a pickle asset to the datasource.
Add a sas asset to the datasource.
Add a spss asset to the datasource.
Add a stata asset to the datasource.
Add a xml asset to the datasource.
Removes the DataAsset referred to by asset_name from internal list of available DataAsset objects.
Parameters
Name Description name
name of DataAsset to be deleted.
Returns the DataAsset referred to by asset_name
Parameters
Name Description name
name of DataAsset sought.
Returns
Type Description great_expectations.datasource.fluent.interfaces._DataAssetT
if named "DataAsset" object exists; otherwise, exception is raised.
Signature
class great_expectations.datasource.fluent.PandasFilesystemDatasource(
*,
type: Literal['pandas_filesystem'] = 'pandas_filesystem',
name: str,
id: Optional[uuid.UUID] = None,
assets: List[great_expectations.datasource.fluent.data_asset.path.file_asset.FileDataAsset] = [],
base_directory: pathlib.Path,
data_context_root_directory: Optional[pathlib.Path] = None
)
Methods
add_csv_asset
Signature
add_csv_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5f0157f0> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5f0158b0> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5f015a00> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5f015d30> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5f015bb0> = None,
sep: typing.Optional[str] = None,
delimiter: typing.Optional[str] = None,
header: Union[int,
Sequence[int],
None,
Literal['infer']] = 'infer',
names: Union[Sequence[str],
None] = None,
index_col: Union[IndexLabel,
Literal[False],
None] = None,
usecols: typing.Optional[typing.Union[int,
str,
typing.Sequence[int]]] = None,
dtype: typing.Optional[dict] = None,
engine: Union[CSVEngine,
None] = None,
true_values: typing.Optional[typing.List] = None,
false_values: typing.Optional[typing.List] = None,
skipinitialspace: bool = False,
skiprows: typing.Optional[typing.Union[typing.Sequence[int],
int]] = None,
skipfooter: int = 0,
nrows: typing.Optional[int] = None,
na_values: Union[str,
Iterable[str],
None] = None,
keep_default_na: bool = True,
na_filter: bool = True,
skip_blank_lines: bool = True,
parse_dates: Union[bool,
Sequence[str],
None] = None,
date_format: typing.Optional[str] = None,
dayfirst: bool = False,
cache_dates: bool = True,
iterator: bool = False,
chunksize: typing.Optional[int] = None,
compression: CompressionOptions = 'infer',
thousands: typing.Optional[str] = None,
decimal: str = '.',
lineterminator: typing.Optional[str] = None,
quotechar: str = '"',
quoting: int = 0,
doublequote: bool = True,
escapechar: typing.Optional[str] = None,
comment: typing.Optional[str] = None,
encoding: typing.Optional[str] = None,
encoding_errors: typing.Optional[str] = 'strict',
dialect: typing.Optional[str] = None,
on_bad_lines: str = 'error',
low_memory: bool = True,
memory_map: bool = False,
storage_options: Union[StorageOptions,
None] = None,
dtype_backend: DtypeBackend = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_excel_asset
Signature
add_excel_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5f0171a0> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5f016f60> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5f016d50> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5f017ce0> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5f017da0> = None,
sheet_name: typing.Optional[typing.Union[str,
int,
typing.List[typing.Union[int,
str]]]] = 0,
header: Union[int,
Sequence[int],
None] = 0,
index_col: Union[int,
str,
Sequence[int],
None] = None,
usecols: typing.Optional[typing.Union[int,
str,
typing.Sequence[int]]] = None,
dtype: typing.Optional[dict] = None,
true_values: Union[Iterable[str],
None] = None,
false_values: Union[Iterable[str],
None] = None,
skiprows: typing.Optional[typing.Union[typing.Sequence[int],
int]] = None,
nrows: typing.Optional[int] = None,
na_values: typing.Any = None,
keep_default_na: bool = True,
na_filter: bool = True,
verbose: bool = False,
parse_dates: typing.Union[typing.List,
typing.Dict,
bool] = False,
date_format: typing.Optional[str] = None,
thousands: typing.Optional[str] = None,
decimal: str = '.',
comment: typing.Optional[str] = None,
skipfooter: int = 0,
storage_options: Union[StorageOptions,
None] = None,
dtype_backend: DtypeBackend = None,
engine_kwargs: typing.Optional[typing.Dict] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_feather_asset
Signature
add_feather_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5f070f50> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5f071070> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5f0711c0> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5f071370> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5f071430> = None,
columns: Union[Sequence[str],
None] = None,
use_threads: bool = True,
storage_options: Union[StorageOptions,
None] = None,
dtype_backend: DtypeBackend = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_fwf_asset
Signature
add_fwf_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5f071b80> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5f071c40> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5f071d90> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5f071f40> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5f072000> = None,
colspecs: Union[Sequence[Tuple[int,
int]],
str,
None] = 'infer',
widths: Union[Sequence[int],
None] = None,
infer_nrows: int = 100,
iterator: bool = False,
chunksize: typing.Optional[int] = None,
kwargs: typing.Optional[dict] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_hdf_asset
Signature
add_hdf_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5f0735f0> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5f0736b0> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5f073800> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5f0739b0> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5f073a70> = None,
key: typing.Any = None,
mode: str = 'r',
errors: str = 'strict',
where: typing.Optional[typing.Union[str,
typing.List]] = None,
start: typing.Optional[int] = None,
stop: typing.Optional[int] = None,
columns: typing.Optional[typing.List[str]] = None,
iterator: bool = False,
chunksize: typing.Optional[int] = None,
kwargs: typing.Optional[dict] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_html_asset
Signature
add_html_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5ef80290> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5ef80350> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5ef804a0> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5ef80650> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5ef80710> = None,
match: Union[str,
Pattern] = '.+',
header: Union[int,
Sequence[int],
None] = None,
index_col: Union[int,
Sequence[int],
None] = None,
skiprows: typing.Optional[typing.Union[typing.Sequence[int],
int]] = None,
attrs: typing.Optional[typing.Dict[str,
str]] = None,
parse_dates: bool = False,
thousands: typing.Optional[str] = ',
',
encoding: typing.Optional[str] = None,
decimal: str = '.',
converters: typing.Optional[typing.Dict] = None,
na_values: Union[Iterable[object],
None] = None,
keep_default_na: bool = True,
displayed_only: bool = True,
dtype_backend: DtypeBackend = None,
storage_options: StorageOptions = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_iceberg_asset
Signature
add_iceberg_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5ef81340> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5ef81400> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5ef81550> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5ef81700> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5ef817c0> = None,
catalog_name: str | None = None,
catalog_properties: dict[str,
typing.Any] | None = None,
columns: list[str] | None = None,
row_filter: str | None = None,
case_sensitive: bool = True,
snapshot_id: int | None = None,
limit: int | None = None,
scan_properties: dict[str,
typing.Any] | None = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_json_asset
Signature
add_json_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5ef81fa0> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5ef82060> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5ef821b0> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5ef82360> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5ef82420> = None,
orient: typing.Optional[str] = None,
typ: Literal['frame',
'series'] = 'frame',
dtype: typing.Optional[dict] = None,
convert_axes: typing.Optional[bool] = None,
convert_dates: typing.Union[bool,
typing.List[str]] = True,
keep_default_dates: bool = True,
precise_float: bool = False,
date_unit: typing.Optional[str] = None,
encoding: typing.Optional[str] = None,
encoding_errors: typing.Optional[str] = 'strict',
lines: bool = False,
chunksize: typing.Optional[int] = None,
compression: CompressionOptions = 'infer',
nrows: typing.Optional[int] = None,
storage_options: Union[StorageOptions,
None] = None,
dtype_backend: DtypeBackend = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_orc_asset
Signature
add_orc_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5ef82f90> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83050> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5ef831a0> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83350> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83410> = None,
columns: typing.Optional[typing.List[str]] = None,
dtype_backend: DtypeBackend = None,
kwargs: typing.Optional[dict] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_parquet_asset
Signature
add_parquet_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83b60> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83c20> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83d70> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5ef83f20> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5edb8050> = None,
engine: str = 'auto',
columns: typing.Optional[typing.List[str]] = None,
storage_options: Union[StorageOptions,
None] = None,
dtype_backend: DtypeBackend = None,
to_pandas_kwargs: typing.Optional[typing.Dict] = None,
kwargs: typing.Optional[dict] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_pickle_asset
Signature
add_pickle_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5edb87d0> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5edb8890> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5edb89e0> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5edb8b90> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5edb8c50> = None,
compression: CompressionOptions = 'infer',
storage_options: Union[StorageOptions,
None] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_sas_asset
Signature
add_sas_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5edb9340> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5edb9400> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5edb9550> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5edb9700> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5edb97c0> = None,
format: typing.Optional[str] = None,
index: typing.Optional[str] = None,
encoding: typing.Optional[str] = None,
chunksize: typing.Optional[int] = None,
iterator: bool = False,
compression: CompressionOptions = 'infer',
**extra_data: typing.Any
) → pydantic.BaseModel
add_spss_asset
Signature
add_spss_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5edb9f70> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5edba030> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5edba180> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5edba330> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5edba3f0> = None,
usecols: typing.Optional[typing.Union[int,
str,
typing.Sequence[int]]] = None,
convert_categoricals: bool = True,
dtype_backend: DtypeBackend = None,
kwargs: typing.Optional[dict] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_stata_asset
Signature
add_stata_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5edbac00> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5edbacc0> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5edbae10> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5edbafc0> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5edbb080> = None,
convert_dates: bool = True,
convert_categoricals: bool = True,
index_col: typing.Optional[str] = None,
convert_missing: bool = False,
preserve_dtypes: bool = True,
columns: Union[Sequence[str],
None] = None,
order_categoricals: bool = True,
chunksize: typing.Optional[int] = None,
iterator: bool = False,
compression: CompressionOptions = 'infer',
storage_options: Union[StorageOptions,
None] = None,
**extra_data: typing.Any
) → pydantic.BaseModel
add_xml_asset
Signature
add_xml_asset(
name: str,
*,
id: <pydantic.v1.fields.DeferredType object at 0x7fda5edbb980> = None,
order_by: <pydantic.v1.fields.DeferredType object at 0x7fda5edbba40> = None,
batch_metadata: <pydantic.v1.fields.DeferredType object at 0x7fda5edbbb90> = None,
batch_definitions: <pydantic.v1.fields.DeferredType object at 0x7fda5edbbd40> = None,
connect_options: <pydantic.v1.fields.DeferredType object at 0x7fda5edbbe00> = None,
xpath: str = './*',
namespaces: typing.Optional[typing.Dict[str,
str]] = None,
elems_only: bool = False,
attrs_only: bool = False,
names: Union[Sequence[str],
None] = None,
dtype: typing.Optional[dict] = None,
encoding: typing.Optional[str] = 'utf-8',
stylesheet: Union[FilePath,
None] = None,
iterparse: typing.Optional[typing.Dict[str,
typing.List[str]]] = None,
compression: CompressionOptions = 'infer',
storage_options: Union[StorageOptions,
None] = None,
dtype_backend: DtypeBackend = None,
**extra_data: typing.Any
) → pydantic.BaseModel
delete_asset
Signature
delete_asset(
name: str
) → None
get_asset
Signature
get_asset(
name: str
) → great_expectations.datasource.fluent.interfaces._DataAssetT