Upgrading datasets 1 to 2
Breaking1.18.4 → 2.21.0Breaking3New12Changed4Fixed67Other13
Fetching 45 releases from GitHub…4 / 45
BreakingUpgrade verdict
3 breaking changes.
In plain terms
- Remove underlines between badges.
- Remove dead code for pyarrow < 15.0.0.
- Remove default trust_remote_code=True.
Must act 3 items need code or config changes
Requires Python >=3.8.0 from 2.14.0 (none stated before).
Adds filelock, pyyaml >=5.1.
Drops responses.
Installs 35 packages in all (was 30).
Also installs anyio, click, h11, httpcore2, httpx2, truststore.
Moves dill 0.4.1 → 0.3.8, fsspec 2026.9.0 → 2024.6.1, huggingface-hub 0.36.2 → 2.4.0, multiprocess 0.70.19 → 0.70.16. Your build may pick other versions.
What you must change
Breaking3
Remove underlines between badges.
Remove dead code for pyarrow < 15.0.0.
Remove default trust_remote_code=True.
trust_remote_code=TrueAPI changes
Generated by DepDelta from published Python source. Not mixed with prose notes.
- removed:
export Array2D - removed:
export Array3D - removed:
export Array4D - removed:
export Array5D - removed:
export ArrowReader - removed:
export ArrowWriter - removed:
export Audio - removed:
export ClassLabel - removed:
export Features - removed:
export Image - removed:
export KeyHasher - removed:
export Sequence - removed:
export Translation - removed:
export TranslationVariableLanguages - removed:
export Value - removed:
export import_main_class - removed:
export prepare_module - removed:
variable SCRIPTS_VERSION - added:
export ExpectedMoreDownloadedFiles - added:
export ExpectedMoreSplits - added:
export UnexpectedDownloadedFile - added:
export UnexpectedSplits - added:
export disable_caching - added:
export enable_caching - added:
export get_dataset_config_info - added:
export get_dataset_default_config_name - added:
export logging
- removed:
class Batch(LazyDict) - removed:
class Example(LazyDict) - removed:
class LazyDict(UserDict) - removed:
constructor LazyDict(data, features=…) - removed:
function concatenate_datasets(dsets: List[Dataset], info: Optional[Any]=…, split: Optional[Any]=…, axis: int=…) - removed:
method Dataset.cast_(features: Features, batch_size: Optional[int]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, num_proc: Optional[int]=…) - removed:
method Dataset.dictionary_encode_column_(column: str) - removed:
method Dataset.flatten_(max_depth=…) - removed:
method Dataset.remove_columns_(column_names: Union[str, List[str]]) - removed:
method Dataset.rename_column_(original_column_name: str, new_column_name: str) - removed:
property DatasetInfoMixin.features -> Features - changed:
function get_indices_from_mask_function(function: Callable, batched: bool, with_indices: bool, input_columns: Optional[Union[str, List[str]]], indices_mapping: Optional[Table]=…, *args, **fn_kwargs) → function get_indices_from_mask_function(function: Callable, batched: bool, with_indices: bool, with_rank: bool, input_columns: Optional[Union[str, List[str]]], indices_mapping: Optional[Table]=…, *args, **fn_kwargs) - changed:
method Dataset.add_faiss_index(column: str, index_name: Optional[str]=…, device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, train_size: Optional[int]=…, faiss_verbose: bool=…, dtype=…) → method Dataset.add_faiss_index(column: str, index_name: Optional[str]=…, device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, batch_size: int=…, train_size: Optional[int]=…, faiss_verbose: bool=…, dtype=…) - changed:
method Dataset.add_faiss_index_from_external_arrays(external_arrays: np.array, index_name: str, device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, train_size: Optional[int]=…, faiss_verbose: bool=…, dtype=…) → method Dataset.add_faiss_index_from_external_arrays(external_arrays: np.array, index_name: str, device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, batch_size: int=…, train_size: Optional[int]=…, faiss_verbose: bool=…, dtype=…) - changed:
method Dataset.cast(features: Features, batch_size: Optional[int]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, num_proc: Optional[int]=…) -> Dataset → method Dataset.cast(features: Features, batch_size: Optional[int]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, num_proc: Optional[int]=…) -> Dataset - changed:
method Dataset.cast_column(column: str, feature: FeatureType, new_fingerprint: str) -> Dataset → method Dataset.cast_column(column: str, feature: FeatureType, new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.filter(function: Optional[Callable]=…, with_indices=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, remove_columns: Optional[List[str]]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…, num_proc: Optional[int]=…, suffix_templat… → method Dataset.filter(function: Optional[Callable]=…, with_indices: bool=…, with_rank: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…, num_proc: Optional[int]=…, suffix_template: s… - changed:
method Dataset.flatten(new_fingerprint, max_depth=…) -> Dataset → method Dataset.flatten(new_fingerprint: Optional[str]=…, max_depth=…) -> Dataset - changed:
method Dataset.flatten_indices(keep_in_memory: bool=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, features: Optional[Features]=…, disable_nullable: bool=…, new_fingerprint: Optional[str]=…) -> Dataset → method Dataset.flatten_indices(keep_in_memory: bool=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, features: Optional[Features]=…, disable_nullable: bool=…, num_proc: Optional[int]=…, new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.map(function: Optional[Callable]=…, with_indices: bool=…, with_rank: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, drop_last_batch: bool=…, remove_columns: Optional[Union[str, List[str]]]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, features: O… → method Dataset.map(function: Optional[Callable]=…, with_indices: bool=…, with_rank: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, drop_last_batch: bool=…, remove_columns: Optional[Union[str, List[str]]]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, f… - changed:
method Dataset.push_to_hub(repo_id: str, split: Optional[str]=…, private: Optional[bool]=…, token: Optional[str]=…, branch: Optional[str]=…, shard_size: Optional[int]=…) → method Dataset.push_to_hub(repo_id: str, config_name: str=…, set_default: Optional[bool]=…, split: Optional[str]=…, data_dir: Optional[str]=…, commit_message: Optional[str]=…, commit_description: Optional[str]=…, private: Optional[bool]=…, token: Optional[str]=…, revision: Optional[str]=…, branch=…, create_pr: Optional[bool]=…, max_shard_size: Optional[Union[int, str]]=…, num_shards: Optional[int… - changed:
method Dataset.remove_columns(column_names: Union[str, List[str]], new_fingerprint) -> Dataset → method Dataset.remove_columns(column_names: Union[str, List[str]], new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.rename_column(original_column_name: str, new_column_name: str, new_fingerprint) -> Dataset → method Dataset.rename_column(original_column_name: str, new_column_name: str, new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.rename_columns(column_mapping: Dict[str, str], new_fingerprint) -> Dataset → method Dataset.rename_columns(column_mapping: Dict[str, str], new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.save_to_disk(dataset_path: str, fs=…) → method Dataset.save_to_disk(dataset_path: PathLike, fs=…, max_shard_size: Optional[Union[str, int]]=…, num_shards: Optional[int]=…, num_proc: Optional[int]=…, storage_options: Optional[dict]=…) - changed:
method Dataset.shuffle(seed: Optional[int]=…, generator: Optional[np.random.Generator]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, indices_cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, new_fingerprint: Optional[str]=…) -> Dataset → method Dataset.shuffle(seed: Optional[int]=…, generator: Optional[np.random.Generator]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, indices_cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.sort(column: str, reverse: bool=…, kind: str=…, null_placement: str=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, indices_cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, new_fingerprint: Optional[str]=…) -> Dataset → method Dataset.sort(column_names: Union[str, Sequence_[str]], reverse: Union[bool, Sequence_[bool]]=…, kind=…, null_placement: str=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, indices_cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, new_fingerprint: Optional[str]=…) -> Dataset - changed:
method Dataset.to_csv(path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, **to_csv_kwargs) -> int → method Dataset.to_csv(path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, storage_options: Optional[dict]=…, **to_csv_kwargs) -> int - changed:
method Dataset.to_dict(batch_size: Optional[int]=…, batched: bool=…) -> Union[dict, Iterator[dict]] → method Dataset.to_dict(batch_size: Optional[int]=…, batched=…) -> Union[dict, Iterator[dict]] - changed:
method Dataset.to_json(path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, **to_json_kwargs) -> int → method Dataset.to_json(path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, storage_options: Optional[dict]=…, **to_json_kwargs) -> int - changed:
method Dataset.to_parquet(path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, **parquet_writer_kwargs) -> int → method Dataset.to_parquet(path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, storage_options: Optional[dict]=…, **parquet_writer_kwargs) -> int - changed:
method Dataset.train_test_split(test_size: Union[float, int, None]=…, train_size: Union[float, int, None]=…, shuffle: bool=…, seed: Optional[int]=…, generator: Optional[np.random.Generator]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, train_indices_cache_file_name: Optional[str]=…, test_indices_cache_file_name: Optional[str]=…, writer_batch_size: Optional[int]=…, train_new_fingerprint… → method Dataset.train_test_split(test_size: Union[float, int, None]=…, train_size: Union[float, int, None]=…, shuffle: bool=…, stratify_by_column: Optional[str]=…, seed: Optional[int]=…, generator: Optional[np.random.Generator]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, train_indices_cache_file_name: Optional[str]=…, test_indices_cache_file_name: Optional[str]=…, writer_bat… - changed:
method Dataset.unique(column: str) -> List[Any] → method Dataset.unique(column: str) -> List - changed:
method TensorflowDatasetMixin.to_tf_dataset(columns: Union[str, List[str]], batch_size: int, shuffle: bool, collate_fn: Callable, drop_remainder: bool=…, collate_fn_args: Dict[str, Any]=…, label_cols: Union[str, List[str]]=…, dummy_labels: bool=…, prefetch: bool=…) → method TensorflowDatasetMixin.to_tf_dataset(batch_size: Optional[int]=…, columns: Optional[Union[str, List[str]]]=…, shuffle: bool=…, collate_fn: Optional[Callable]=…, drop_remainder: bool=…, collate_fn_args: Optional[Dict[str, Any]]=…, label_cols: Optional[Union[str, List[str]]]=…, prefetch: bool=…, num_workers: int=…, num_test_batches: int=…) - changed:
static method Dataset.from_csv(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) → static method Dataset.from_csv(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, num_proc: Optional[int]=…, **kwargs) - changed:
static method Dataset.from_dict(mapping: dict, features: Optional[Features]=…, info: Optional[Any]=…, split: Optional[Any]=…) -> Dataset → static method Dataset.from_dict(mapping: dict, features: Optional[Features]=…, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…) -> Dataset - changed:
static method Dataset.from_json(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, field: Optional[str]=…, **kwargs) → static method Dataset.from_json(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, field: Optional[str]=…, num_proc: Optional[int]=…, **kwargs) - changed:
static method Dataset.from_parquet(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, columns: Optional[List[str]]=…, **kwargs) → static method Dataset.from_parquet(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, columns: Optional[List[str]]=…, num_proc: Optional[int]=…, **kwargs) - changed:
static method Dataset.from_text(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) → static method Dataset.from_text(path_or_paths: Union[PathLike, List[PathLike]], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, num_proc: Optional[int]=…, **kwargs) - changed:
static method Dataset.load_from_disk(dataset_path: str, fs=…, keep_in_memory: Optional[bool]=…) -> Dataset → static method Dataset.load_from_disk(dataset_path: PathLike, fs=…, keep_in_memory: Optional[bool]=…, storage_options: Optional[dict]=…) -> Dataset - added:
method Dataset.batch(batch_size: int, drop_last_batch: bool=…, num_proc: Optional[int]=…, new_fingerprint: Optional[str]=…) -> Dataset - added:
method Dataset.iter(batch_size: int, drop_last_batch: bool=…) - added:
method Dataset.select_columns(column_names: Union[str, List[str]], new_fingerprint: Optional[str]=…) -> Dataset - added:
method Dataset.skip(n: int) -> Dataset - added:
method Dataset.take(n: int) -> Dataset - added:
method Dataset.to_iterable_dataset(num_shards: Optional[int]=…) -> IterableDataset - added:
method Dataset.to_list() -> list - added:
method Dataset.to_polars(batch_size: Optional[int]=…, batched: bool=…, schema_overrides: Optional[dict]=…, rechunk: bool=…) -> Union[pl.DataFrame, Iterator[pl.DataFrame]] - added:
method Dataset.to_sql(name: str, con: Union[str, sqlalchemy.engine.Connection, sqlalchemy.engine.Engine, sqlite3.Connection], batch_size: Optional[int]=…, **sql_writer_kwargs) -> int - added:
property Dataset.features -> Features - added:
property DatasetInfoMixin.features -> Optional[Features] - added:
static method Dataset.from_generator(generator: Callable, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, gen_kwargs: Optional[dict]=…, num_proc: Optional[int]=…, split: NamedSplit=…, **kwargs) - added:
static method Dataset.from_list(mapping: List[dict], features: Optional[Features]=…, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…) -> Dataset - added:
static method Dataset.from_polars(df: pl.DataFrame, features: Optional[Features]=…, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…) -> Dataset - added:
static method Dataset.from_spark(df: pyspark.sql.DataFrame, split: Optional[NamedSplit]=…, features: Optional[Features]=…, keep_in_memory: bool=…, cache_dir: str=…, working_dir: str=…, load_from_cache_file: bool=…, **kwargs) - added:
static method Dataset.from_sql(sql: Union[str, sqlalchemy.sql.Selectable], con: Union[str, sqlalchemy.engine.Connection, sqlalchemy.engine.Engine, sqlite3.Connection], features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) - added:
variable PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED
- changed:
function make_file_instructions(name, split_infos, instruction, filetype_suffix=…) → function make_file_instructions(name: str, split_infos: List[SplitInfo], instruction: Union[str, ReadInstruction], filetype_suffix: Optional[str]=…, prefix_path: Optional[str]=…) -> FileInstructions
- changed:
constructor ArrowWriter(schema: Optional[pa.Schema]=…, features: Optional[Features]=…, path: Optional[str]=…, stream: Optional[pa.NativeFile]=…, fingerprint: Optional[str]=…, writer_batch_size: Optional[int]=…, hash_salt: Optional[str]=…, check_duplicates: Optional[bool]=…, disable_nullable: bool=…, update_features: bool=…, with_metadata: bool=…, unit: str=…) → constructor ArrowWriter(schema: Optional[pa.Schema]=…, features: Optional[Features]=…, path: Optional[str]=…, stream: Optional[pa.NativeFile]=…, fingerprint: Optional[str]=…, writer_batch_size: Optional[int]=…, hash_salt: Optional[str]=…, check_duplicates: Optional[bool]=…, disable_nullable: bool=…, update_features: bool=…, with_metadata: bool=…, unit: str=…, embed_local_files: bool=…, storage_op… - changed:
function parquet_to_arrow(sources, destination) → function parquet_to_arrow(source, destination) -> List[int] - changed:
method ArrowWriter.write_batch(batch_examples: Dict[str, List[Any]], writer_batch_size: Optional[int]=…) → method ArrowWriter.write_batch(batch_examples: Dict[str, List], writer_batch_size: Optional[int]=…) - added:
class ParquetWriter(ArrowWriter) - added:
class SchemaInferenceError(ValueError) - added:
function get_parquet_lengths(sources) -> List[int]
- removed:
class DatasetBuildError(Exception) - removed:
class ManualDownloadError(DatasetBuildError) - removed:
constructor GeneratorBasedBuilder(*args, writer_batch_size=…, **kwargs) - removed:
field ArrowBasedBuilder.test_dummy_data - removed:
field BeamBasedBuilder.test_dummy_data - removed:
field GeneratorBasedBuilder.DEFAULT_WRITER_BATCH_SIZE - removed:
field GeneratorBasedBuilder.test_dummy_data - changed:
constructor BeamBasedBuilder(*args, **kwargs) → constructor BeamBasedBuilder(*args, beam_runner=…, beam_options=…, **kwargs) - changed:
constructor DatasetBuilder(cache_dir: Optional[str]=…, name: Optional[str]=…, hash: Optional[str]=…, base_path: Optional[str]=…, info: Optional[DatasetInfo]=…, features: Optional[Features]=…, use_auth_token: Optional[Union[bool, str]]=…, namespace: Optional[str]=…, data_files: Optional[Union[str, list, dict, DataFilesDict]]=…, data_dir: Optional[str]=…, **config_kwargs) → constructor DatasetBuilder(cache_dir: Optional[str]=…, dataset_name: Optional[str]=…, config_name: Optional[str]=…, hash: Optional[str]=…, base_path: Optional[str]=…, info: Optional[DatasetInfo]=…, features: Optional[Features]=…, token: Optional[Union[bool, str]]=…, use_auth_token=…, repo_id: Optional[str]=…, data_files: Optional[Union[str, list, dict, DataFilesDict]]=…, data_dir: Optional[str]=…… - changed:
field BuilderConfig.data_files: Optional[DataFilesDict] → field BuilderConfig.data_files: Optional[Union[DataFilesDict, DataFilesPatternsDict]] - changed:
field BuilderConfig.version: Optional[Union[str, utils.Version]] → field BuilderConfig.version: Optional[Union[utils.Version, str]] - changed:
method DatasetBuilder.as_dataset(split: Optional[Split]=…, run_post_process=…, ignore_verifications=…, in_memory=…) -> Union[Dataset, DatasetDict] → method DatasetBuilder.as_dataset(split: Optional[Split]=…, run_post_process=…, verification_mode: Optional[Union[VerificationMode, str]]=…, ignore_verifications=…, in_memory=…) -> Union[Dataset, DatasetDict] - changed:
method DatasetBuilder.as_streaming_dataset(split: Optional[str]=…, base_path: Optional[str]=…, use_auth_token: Optional[str]=…) -> Union[Dict[str, IterableDataset], IterableDataset] → method DatasetBuilder.as_streaming_dataset(split: Optional[str]=…, base_path: Optional[str]=…) -> Union[Dict[str, IterableDataset], IterableDataset] - changed:
method DatasetBuilder.download_and_prepare(download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, ignore_verifications: bool=…, try_from_hf_gcs: bool=…, dl_manager: Optional[DownloadManager]=…, base_path: Optional[str]=…, use_auth_token: Optional[Union[bool, str]]=…, **download_and_prepare_kwargs) → method DatasetBuilder.download_and_prepare(output_dir: Optional[str]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, verification_mode: Optional[Union[VerificationMode, str]]=…, ignore_verifications=…, try_from_hf_gcs=…, dl_manager: Optional[DownloadManager]=…, base_path: Optional[str]=…, use_auth_token=…, file_format: str=…, max_shard_size: Op… - changed:
static method DatasetBuilder.builder_configs() → static method DatasetBuilder.builder_configs() -> Dict[str, BuilderConfig] - changed:
static method DatasetBuilder.get_all_exported_dataset_infos() -> dict → static method DatasetBuilder.get_all_exported_dataset_infos() -> DatasetInfosDict - added:
field DatasetBuilder.DEFAULT_WRITER_BATCH_SIZE - added:
method BeamBasedBuilder.as_streaming_dataset(split: Optional[str]=…) -> Union[Dict[str, IterableDataset], IterableDataset]
- changed:
function interleave_datasets(datasets: List[DatasetType], probabilities: Optional[List[float]]=…, seed: Optional[int]=…) -> DatasetType → function interleave_datasets(datasets: List[DatasetType], probabilities: Optional[List[float]]=…, seed: Optional[int]=…, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…, stopping_strategy: Literal['first_exhausted', 'all_exhausted']=…) -> DatasetType - added:
function concatenate_datasets(dsets: List[DatasetType], info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…, axis: int=…) -> DatasetType
- added:
class ConvertToParquetCommand(BaseDatasetsCLICommand) - added:
constructor ConvertToParquetCommand(dataset_id: str, token: Optional[str], revision: Optional[str], trust_remote_code: bool) - added:
method ConvertToParquetCommand.run() -> None - added:
static method ConvertToParquetCommand.register_subcommand(parser)
- added:
function parse_unknown_args(unknown_args)
- added:
class DeleteFromHubCommand(BaseDatasetsCLICommand) - added:
constructor DeleteFromHubCommand(dataset_id: str, config_name: str, token: Optional[str], revision: Optional[str]) - added:
method DeleteFromHubCommand.run() -> None - added:
static method DeleteFromHubCommand.register_subcommand(parser)
- removed:
function test_command_factory(args) - added:
function dummy_data_command_factory(args)
- changed:
constructor RunBeamCommand(dataset: str, name: str, cache_dir: str, beam_pipeline_options: str, data_dir: str, all_configs: bool, save_infos: bool, ignore_verifications: bool, force_redownload: bool) → constructor RunBeamCommand(dataset: str, name: str, cache_dir: str, beam_pipeline_options: str, data_dir: str, all_configs: bool, save_infos: bool, ignore_verifications: bool, force_redownload: bool, **config_kwargs) - changed:
function run_beam_command_factory(args) → function run_beam_command_factory(args, **kwargs)
- added:
variable ARROW_READER_BATCH_SIZE_IN_DATASET_ITER - added:
variable DILL_VERSION - added:
variable ENV_VARS_FALSE_AND_AUTO_VALUES - added:
variable ENV_VARS_FALSE_VALUES - added:
variable FSSPEC_VERSION - added:
variable HF_DATASETS_DISABLE_PROGRESS_BARS: Optional[bool] - added:
variable HF_DATASETS_MULTITHREADING_MAX_WORKERS - added:
variable HF_DATASETS_TRUST_REMOTE_CODE: Optional[bool] - added:
variable HF_HUB_OFFLINE - added:
variable HF_HUB_VERSION - added:
variable HUB_DATASETS_HFFS_URL - added:
variable IS_MP3_SUPPORTED - added:
variable IS_OPUS_SUPPORTED - added:
variable MAX_SHARD_SIZE - added:
variable METADATA_CONFIGS_FIELD - added:
variable PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS - added:
variable PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS - added:
variable PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS - added:
variable PBAR_REFRESH_TIME_INTERVAL - added:
variable POLARS_AVAILABLE - added:
variable POLARS_VERSION - added:
variable PY7ZR_AVAILABLE - added:
variable REPOCARD_FILENAME - added:
variable REPOYAML_FILENAME - added:
variable SQLALCHEMY_AVAILABLE - added:
variable TEMP_CACHE_DIR_PREFIX - added:
variable TIME_OUT_REMOTE_CODE - added:
variable UPLOADS_MAX_NUMBER_PER_COMMIT - added:
variable USE_PARQUET_EXPORT
- removed:
function get_patterns_in_dataset_repository(dataset_info: huggingface_hub.hf_api.DatasetInfo) -> Dict[str, List[str]] - removed:
function get_patterns_locally(base_path: str) -> Dict[str, List[str]] - removed:
function resolve_patterns_in_dataset_repository(dataset_info: huggingface_hub.hf_api.DatasetInfo, patterns: List[str], allowed_extensions: Optional[list]=…) -> List[Url] - removed:
function resolve_patterns_locally_or_by_urls(base_path: str, patterns: List[str], allowed_extensions: Optional[List[str]]=…) -> List[Union[Path, Url]] - removed:
variable DEFAULT_SPLIT - changed:
class DataFilesList(List[Union[Path, Url]]) → class DataFilesList(List[str]) - changed:
constructor DataFilesList(data_files: List[Union[Path, Url]], origin_metadata: List[Tuple[str]]) → constructor DataFilesList(data_files: List[str], origin_metadata: List[SingleOriginMetadata]) -> None - changed:
static method DataFilesDict.from_hf_repo(patterns: Dict[str, Union[List[str], DataFilesList]], dataset_info: huggingface_hub.hf_api.DatasetInfo, allowed_extensions: Optional[List[str]]=…) -> DataFilesDict → static method DataFilesDict.from_hf_repo(patterns: Dict[str, Union[List[str], DataFilesList]], dataset_info: huggingface_hub.hf_api.DatasetInfo, base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> DataFilesDict - changed:
static method DataFilesDict.from_local_or_remote(patterns: Dict[str, Union[List[str], DataFilesList]], base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, use_auth_token: Optional[Union[bool, str]]=…) -> DataFilesDict → static method DataFilesDict.from_local_or_remote(patterns: Dict[str, Union[List[str], DataFilesList]], base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> DataFilesDict - changed:
static method DataFilesList.from_hf_repo(patterns: List[str], dataset_info: huggingface_hub.hf_api.DatasetInfo, allowed_extensions: Optional[List[str]]=…) -> DataFilesList → static method DataFilesList.from_hf_repo(patterns: List[str], dataset_info: huggingface_hub.hf_api.DatasetInfo, base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> DataFilesList - changed:
static method DataFilesList.from_local_or_remote(patterns: List[str], base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, use_auth_token: Optional[Union[bool, str]]=…) -> DataFilesList → static method DataFilesList.from_local_or_remote(patterns: List[str], base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> DataFilesList - added:
class DataFilesPatternsDict(Dict[str, DataFilesPatternsList]) - added:
class DataFilesPatternsList(List[str]) - added:
class EmptyDatasetError(FileNotFoundError) - added:
constructor DataFilesPatternsList(patterns: List[str], allowed_extensions: List[Optional[List[str]]]) - added:
function get_data_patterns(base_path: str, download_config: Optional[DownloadConfig]=…) -> Dict[str, List[str]] - added:
function get_metadata_patterns(base_path: str, download_config: Optional[DownloadConfig]=…) -> List[str] - added:
function resolve_pattern(pattern: str, base_path: str, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> List[str] - added:
method DataFilesDict.filter_extensions(extensions: List[str]) -> DataFilesDict - added:
method DataFilesList.filter_extensions(extensions: List[str]) -> DataFilesList - added:
method DataFilesPatternsDict.filter_extensions(extensions: List[str]) -> DataFilesPatternsDict - added:
method DataFilesPatternsDict.resolve(base_path: str, download_config: Optional[DownloadConfig]=…) -> DataFilesDict - added:
method DataFilesPatternsList.filter_extensions(extensions: List[str]) -> DataFilesPatternsList - added:
method DataFilesPatternsList.resolve(base_path: str, download_config: Optional[DownloadConfig]=…) -> DataFilesList - added:
static method DataFilesDict.from_patterns(patterns: Dict[str, Union[List[str], DataFilesList]], base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> DataFilesDict - added:
static method DataFilesList.from_patterns(patterns: List[str], base_path: Optional[str]=…, allowed_extensions: Optional[List[str]]=…, download_config: Optional[DownloadConfig]=…) -> DataFilesList - added:
static method DataFilesPatternsDict.from_patterns(patterns: Dict[str, List[str]], allowed_extensions: Optional[List[str]]=…) -> DataFilesPatternsDict - added:
static method DataFilesPatternsList.from_patterns(patterns: List[str], allowed_extensions: Optional[List[str]]=…) -> DataFilesPatternsList - added:
variable DEFAULT_SPLITS - added:
variable KEYWORDS_IN_DIR_NAME_BASE_PATTERNS - added:
variable KEYWORDS_IN_FILENAME_BASE_PATTERNS - added:
variable METADATA_PATTERNS - added:
variable NON_WORDS_CHARS - added:
variable SANITIZED_DEFAULT_SPLIT - added:
variable SPLIT_KEYWORDS - added:
variable SingleOriginMetadata
- removed:
method DatasetDict.cast_(features: Features) - removed:
method DatasetDict.dictionary_encode_column_(column: str) - removed:
method DatasetDict.flatten_(max_depth=…) - removed:
method DatasetDict.remove_columns_(column_names: Union[str, List[str]]) - removed:
method DatasetDict.rename_column_(original_column_name: str, new_column_name: str) - changed:
method DatasetDict.filter(function, with_indices=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, remove_columns: Optional[List[str]]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…, num_proc: Optional[int]=…, desc: Opti… → method DatasetDict.filter(function: Optional[Callable]=…, with_indices: bool=…, with_rank: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…, num_proc: Optional[… - changed:
method DatasetDict.map(function, with_indices: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, remove_columns: Optional[Union[str, List[str]]]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…, features: Optional[Features]=…, disable_nullable: b… → method DatasetDict.map(function: Optional[Callable]=…, with_indices: bool=…, with_rank: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, drop_last_batch: bool=…, remove_columns: Optional[Union[str, List[str]]]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batc… - changed:
method DatasetDict.push_to_hub(repo_id, private: Optional[bool]=…, token: Optional[str]=…, branch: Optional[None]=…, shard_size: Optional[int]=…) → method DatasetDict.push_to_hub(repo_id, config_name: str=…, set_default: Optional[bool]=…, data_dir: Optional[str]=…, commit_message: Optional[str]=…, commit_description: Optional[str]=…, private: Optional[bool]=…, token: Optional[str]=…, revision: Optional[str]=…, branch=…, create_pr: Optional[bool]=…, max_shard_size: Optional[Union[int, str]]=…, num_shards: Optional[Dict[str, int]]=…, embed_ext… - changed:
method DatasetDict.save_to_disk(dataset_dict_path: str, fs=…) → method DatasetDict.save_to_disk(dataset_dict_path: PathLike, fs=…, max_shard_size: Optional[Union[str, int]]=…, num_shards: Optional[Dict[str, int]]=…, num_proc: Optional[int]=…, storage_options: Optional[dict]=…) - changed:
method DatasetDict.shuffle(seeds: Optional[Union[int, Dict[str, Optional[int]]]]=…, seed: Optional[int]=…, generators: Optional[Dict[str, np.random.Generator]]=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, indices_cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…) -> DatasetDict → method DatasetDict.shuffle(seeds: Optional[Union[int, Dict[str, Optional[int]]]]=…, seed: Optional[int]=…, generators: Optional[Dict[str, np.random.Generator]]=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, indices_cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…) -> DatasetDict - changed:
method DatasetDict.sort(column: str, reverse: bool=…, kind: str=…, null_placement: str=…, keep_in_memory: bool=…, load_from_cache_file: bool=…, indices_cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…) -> DatasetDict → method DatasetDict.sort(column_names: Union[str, Sequence[str]], reverse: Union[bool, Sequence[bool]]=…, kind=…, null_placement: str=…, keep_in_memory: bool=…, load_from_cache_file: Optional[bool]=…, indices_cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…) -> DatasetDict - changed:
method DatasetDict.unique(column: str) -> Dict[str, List[Any]] → method DatasetDict.unique(column: str) -> Dict[str, List] - changed:
static method DatasetDict.load_from_disk(dataset_dict_path: str, fs=…, keep_in_memory: Optional[bool]=…) -> DatasetDict → static method DatasetDict.load_from_disk(dataset_dict_path: PathLike, fs=…, keep_in_memory: Optional[bool]=…, storage_options: Optional[dict]=…) -> DatasetDict - added:
method DatasetDict.flatten_indices(keep_in_memory: bool=…, cache_file_names: Optional[Dict[str, Optional[str]]]=…, writer_batch_size: Optional[int]=…, features: Optional[Features]=…, disable_nullable: bool=…, num_proc: Optional[int]=…, new_fingerprint: Optional[str]=…) -> DatasetDict - added:
method DatasetDict.rename_columns(column_mapping: Dict[str, str]) -> DatasetDict - added:
method DatasetDict.select_columns(column_names: Union[str, List[str]]) -> DatasetDict - added:
method IterableDatasetDict.cast(features: Features) -> IterableDatasetDict - added:
method IterableDatasetDict.cast_column(column: str, feature: FeatureType) -> IterableDatasetDict - added:
method IterableDatasetDict.filter(function: Optional[Callable]=…, with_indices=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…) -> IterableDatasetDict - added:
method IterableDatasetDict.map(function: Optional[Callable]=…, with_indices: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: int=…, drop_last_batch: bool=…, remove_columns: Optional[Union[str, List[str]]]=…, fn_kwargs: Optional[dict]=…) -> IterableDatasetDict - added:
method IterableDatasetDict.remove_columns(column_names: Union[str, List[str]]) -> IterableDatasetDict - added:
method IterableDatasetDict.rename_column(original_column_name: str, new_column_name: str) -> IterableDatasetDict - added:
method IterableDatasetDict.rename_columns(column_mapping: Dict[str, str]) -> IterableDatasetDict - added:
method IterableDatasetDict.select_columns(column_names: Union[str, List[str]]) -> IterableDatasetDict - added:
method IterableDatasetDict.shuffle(seed=…, generator: Optional[np.random.Generator]=…, buffer_size: int=…) -> IterableDatasetDict - added:
method IterableDatasetDict.with_format(type: Optional[str]=…) -> IterableDatasetDict
- added:
function split_dataset_by_node(dataset: DatasetType, rank: int, world_size: int) -> DatasetType - added:
variable DatasetType
- added:
export DownloadConfig - added:
export DownloadManager - added:
export DownloadMode - added:
export StreamingDownloadManager
- added:
class DownloadConfig - added:
field DownloadConfig.cache_dir: Optional[Union[str, Path]] - added:
field DownloadConfig.delete_extracted: bool - added:
field DownloadConfig.disable_tqdm: bool - added:
field DownloadConfig.download_desc: Optional[str] - added:
field DownloadConfig.extract_compressed_file: bool - added:
field DownloadConfig.extract_on_the_fly: bool - added:
field DownloadConfig.force_download: bool - added:
field DownloadConfig.force_extract: bool - added:
field DownloadConfig.ignore_url_params: bool - added:
field DownloadConfig.local_files_only: bool - added:
field DownloadConfig.max_retries: int - added:
field DownloadConfig.num_proc: Optional[int] - added:
field DownloadConfig.proxies: Optional[Dict] - added:
field DownloadConfig.resume_download: bool - added:
field DownloadConfig.storage_options: Dict[str, Any] - added:
field DownloadConfig.token: Optional[Union[str, bool]] - added:
field DownloadConfig.use_auth_token: InitVar[Optional[Union[str, bool]]] - added:
field DownloadConfig.use_etag: bool - added:
field DownloadConfig.user_agent: Optional[str] - added:
method DownloadConfig.copy() -> DownloadConfig
- added:
class DownloadManager - added:
class DownloadMode(enum.Enum) - added:
class GenerateMode(DeprecatedEnum) - added:
constructor DownloadManager(dataset_name: Optional[str]=…, data_dir: Optional[str]=…, download_config: Optional[DownloadConfig]=…, base_path: Optional[str]=…, record_checksums=…) - added:
field DownloadManager.is_streaming - added:
field DownloadMode.FORCE_REDOWNLOAD - added:
field DownloadMode.REUSE_CACHE_IF_EXISTS - added:
field DownloadMode.REUSE_DATASET_IF_EXISTS - added:
field GenerateMode.FORCE_REDOWNLOAD - added:
field GenerateMode.REUSE_CACHE_IF_EXISTS - added:
field GenerateMode.REUSE_DATASET_IF_EXISTS - added:
method DownloadManager.delete_extracted_files() - added:
method DownloadManager.download(url_or_urls) - added:
method DownloadManager.download_and_extract(url_or_urls) - added:
method DownloadManager.download_custom(url_or_urls, custom_download) - added:
method DownloadManager.extract(path_or_paths, num_proc=…) - added:
method DownloadManager.get_recorded_sizes_checksums() - added:
method DownloadManager.iter_archive(path_or_buf: Union[str, io.BufferedReader]) - added:
method DownloadManager.iter_files(paths: Union[str, List[str]]) - added:
method DownloadManager.manage_extracted_files() - added:
property DownloadManager.downloaded_size - added:
property DownloadManager.manual_dir - added:
property GenerateMode.help_message - added:
static method DownloadManager.ship_files_with_pipeline(downloaded_path_or_paths, pipeline) - added:
variable logger
- added:
class MockDownloadManager - added:
constructor MockDownloadManager(dataset_name: str, config: str, version: Union[Version, str], cache_dir: Optional[str]=…, use_local_dummy_data: bool=…, load_existing_dummy_data: bool=…, download_callbacks: Optional[List[Callable]]=…) - added:
field MockDownloadManager.datasets_scripts_dir - added:
field MockDownloadManager.dummy_file_name - added:
field MockDownloadManager.is_streaming - added:
method MockDownloadManager.create_dummy_data_dict(path_to_dummy_data, data_url) - added:
method MockDownloadManager.create_dummy_data_list(path_to_dummy_data, data_url) - added:
method MockDownloadManager.create_dummy_data_single(path_to_dummy_data, data_url) - added:
method MockDownloadManager.delete_extracted_files() - added:
method MockDownloadManager.download(data_url, *args) - added:
method MockDownloadManager.download_and_extract(data_url, *args) - added:
method MockDownloadManager.download_custom(data_url, custom_download) - added:
method MockDownloadManager.download_dummy_data() - added:
method MockDownloadManager.extract(path, *args, **kwargs) - added:
method MockDownloadManager.get_recorded_sizes_checksums() - added:
method MockDownloadManager.iter_archive(path) - added:
method MockDownloadManager.iter_files(paths) - added:
method MockDownloadManager.manage_extracted_files() - added:
property MockDownloadManager.dummy_data_folder - added:
property MockDownloadManager.dummy_file - added:
property MockDownloadManager.dummy_zip_file - added:
property MockDownloadManager.github_path_to_dummy_data - added:
property MockDownloadManager.local_path_to_dummy_data - added:
property MockDownloadManager.manual_dir - added:
variable logger
- added:
class StreamingDownloadManager - added:
constructor StreamingDownloadManager(dataset_name: Optional[str]=…, data_dir: Optional[str]=…, download_config: Optional[DownloadConfig]=…, base_path: Optional[str]=…) - added:
field StreamingDownloadManager.is_streaming - added:
method StreamingDownloadManager.download(url_or_urls) - added:
method StreamingDownloadManager.download_and_extract(url_or_urls) - added:
method StreamingDownloadManager.extract(url_or_urls) - added:
method StreamingDownloadManager.iter_archive(urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[Tuple] - added:
method StreamingDownloadManager.iter_files(urlpaths: Union[str, List[str]]) -> Iterable[str] - added:
property StreamingDownloadManager.manual_dir - added:
variable logger
- added:
class ChecksumVerificationError(DatasetsError, ChecksumVerificationException) - added:
class ChecksumVerificationException(Exception) - added:
class DataFilesNotFoundError(FileNotFoundDatasetsError) - added:
class DatasetBuildError(DatasetsError) - added:
class DatasetGenerationCastError(DatasetGenerationError) - added:
class DatasetGenerationError(DatasetBuildError) - added:
class DatasetNotFoundError(FileNotFoundDatasetsError) - added:
class DatasetsError(Exception) - added:
class DefunctDatasetError(DatasetsError) - added:
class ExpectedMoreDownloadedFiles(ChecksumVerificationException) - added:
class ExpectedMoreDownloadedFilesError(ChecksumVerificationError, ExpectedMoreDownloadedFiles) - added:
class ExpectedMoreSplits(SplitsVerificationException) - added:
class ExpectedMoreSplitsError(SplitsVerificationError, ExpectedMoreSplits) - added:
class FileFormatError(DatasetBuildError) - added:
class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError) - added:
class ManualDownloadError(DatasetBuildError) - added:
class NonMatchingChecksumError(ChecksumVerificationError) - added:
class NonMatchingSplitsSizesError(SplitsVerificationError) - added:
class SplitsVerificationError(DatasetsError, SplitsVerificationException) - added:
class SplitsVerificationException(Exception) - added:
class UnexpectedDownloadedFile(ChecksumVerificationException) - added:
class UnexpectedDownloadedFileError(ChecksumVerificationError, UnexpectedDownloadedFile) - added:
class UnexpectedSplits(SplitsVerificationException) - added:
class UnexpectedSplitsError(SplitsVerificationError, UnexpectedSplits) - added:
constructor ChecksumVerificationError(*args, **kwargs) - added:
constructor SplitsVerificationError(*args, **kwargs) - added:
static method DatasetGenerationCastError.from_cast_error(cast_error: CastError, builder_name: str, gen_kwargs: Dict[str, Any], token: Optional[Union[bool, str]]) -> DatasetGenerationCastError
- removed:
export objects_to_list_of_image_dicts - added:
export Array2D - added:
export Array3D - added:
export Array4D - added:
export Array5D - added:
export ClassLabel - added:
export Features - added:
export LargeList - added:
export Sequence - added:
export Value
- changed:
method Audio.decode_example(value: dict) -> dict → method Audio.decode_example(value: dict, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]]=…) -> dict - changed:
method Audio.encode_example(value: Union[str, dict]) -> dict → method Audio.encode_example(value: Union[str, bytes, dict]) -> dict - added:
method Audio.embed_storage(storage: pa.StructArray) -> pa.StructArray - added:
method Audio.flatten() -> Union[FeatureType, Dict[str, FeatureType]]
- removed:
method ArrayExtensionArray.to_list_of_numpy(zero_copy_only=…) - changed:
field ClassLabel.num_classes: int → field ClassLabel.num_classes: InitVar[Optional[int]] - changed:
function cast_to_python_objects(obj: Any, only_1d_for_numpy=…) -> Any → function cast_to_python_objects(obj: Any, only_1d_for_numpy=…, optimize_list_casting=…) -> Any - changed:
function decode_nested_example(schema, obj) → function decode_nested_example(schema, obj, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]]=…) - changed:
function encode_nested_example(schema, obj) → function encode_nested_example(schema, obj, level=…) - changed:
function require_decoding(feature: FeatureType) -> bool → function require_decoding(feature: FeatureType, ignore_decode_attribute: bool=…) -> bool - changed:
method ClassLabel.int2str(values: Union[int, Iterable]) → method ClassLabel.int2str(values: Union[int, Iterable]) -> Union[str, Iterable] - changed:
method ClassLabel.str2int(values: Union[str, Iterable]) → method ClassLabel.str2int(values: Union[str, Iterable]) -> Union[int, Iterable] - changed:
method Features.decode_batch(batch: dict) → method Features.decode_batch(batch: dict, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]]=…) - changed:
method Features.decode_example(example: dict) → method Features.decode_example(example: dict, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]]=…) - added:
class LargeList - added:
field Features.clear - added:
field Features.pop - added:
field Features.popitem - added:
field Features.setdefault - added:
field Features.update - added:
field LargeList.dtype: Any - added:
field LargeList.id: Optional[str] - added:
field LargeList.pa_type: ClassVar[Any] - added:
function any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, List], type: pa.DataType=…) -> pa.ListArray - added:
function contains_any_np_array(data: Any) - added:
function keep_features_dicts_synced(func) - added:
function register_feature(feature_cls: type, feature_type: str) - added:
function require_storage_cast(feature: FeatureType) -> bool - added:
function require_storage_embed(feature: FeatureType) -> bool - added:
function to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array - added:
method ClassLabel.cast_storage(storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array - added:
method Features.encode_column(column, column_name: str)
- changed:
method Image.decode_example(value: dict) -> PIL.Image.Image → method Image.decode_example(value: dict, token_per_repo_id=…) -> PIL.Image.Image - changed:
method Image.encode_example(value: Union[str, dict, np.ndarray, PIL.Image.Image]) -> dict → method Image.encode_example(value: Union[str, bytes, dict, np.ndarray, PIL.Image.Image]) -> dict - added:
field Image.mode: Optional[str] - added:
function encode_np_array(array: np.ndarray) -> dict - added:
method Image.embed_storage(storage: pa.StructArray) -> pa.StructArray - added:
method Image.flatten() -> Union[FeatureType, Dict[str, FeatureType]]
- added:
method Translation.flatten() -> Union[FeatureType, Dict[str, FeatureType]] - added:
method TranslationVariableLanguages.flatten() -> Union[FeatureType, Dict[str, FeatureType]]
- removed:
export HfFileSystem - added:
export deprecated - added:
function rename(fs: fsspec.AbstractFileSystem, src: str, dst: str)
- removed:
constructor ZstdFileSystem(fo: str, mode: str=…, target_protocol: Optional[str]=…, target_options: Optional[dict]=…, block_size: int=…, **kwargs)
- removed:
class HfFileSystem(AbstractFileSystem) - removed:
constructor HfFileSystem(repo_info: Optional[DatasetInfo]=…, token: Optional[str]=…, **kwargs) - removed:
field HfFileSystem.protocol - removed:
field HfFileSystem.root_marker - removed:
method HfFileSystem.info(path, **kwargs) - removed:
method HfFileSystem.ls(path, detail=…, **kwargs)
- changed:
function generate_fingerprint(dataset) -> str → function generate_fingerprint(dataset: Dataset) -> str - changed:
function generate_random_fingerprint(nbits=…) -> str → function generate_random_fingerprint(nbits: int=…) -> str - added:
function disable_caching() - added:
function enable_caching() - added:
function format_kwargs_for_fingerprint(func: Callable, args: Tuple, kwargs: Dict[str, Any], use_kwargs: Optional[List[str]]=…, ignore_kwargs: Optional[List[str]]=…, randomized_function: bool=…) -> Dict[str, Any] - added:
function format_transform_for_fingerprint(func: Callable, version: Optional[str]=…) -> str - added:
function validate_fingerprint(fingerprint: str, max_length=…) - added:
variable fingerprint_rng
- removed:
export get_logger - added:
export PolarsFormatter - added:
export TensorFormatter - added:
export logging
- removed:
class NumpyFormatter(Formatter[dict, np.ndarray, dict]) - removed:
constructor NumpyFormatter(features=…, decoded=…, **np_array_kwargs) - removed:
method NumpyFormatter.format_batch(pa_table: pa.Table) -> dict - removed:
method NumpyFormatter.format_column(pa_table: pa.Table) -> np.ndarray - removed:
method NumpyFormatter.format_row(pa_table: pa.Table) -> dict - changed:
class PandasFormatter(Formatter) → class PandasFormatter(Formatter[pd.DataFrame, pd.Series, pd.DataFrame]) - changed:
class PythonFormatter(Formatter[dict, list, dict]) → class PythonFormatter(Formatter[Mapping, list, Mapping]) - changed:
constructor CustomFormatter(transform: Callable[[dict], dict], features=…, decoded=…, **kwargs) → constructor CustomFormatter(transform: Callable[[dict], dict], features=…, **kwargs) - changed:
constructor Formatter(features=…, decoded=…) → constructor Formatter(features: Optional[Features]=…) - changed:
constructor PandasFeaturesDecoder(features) → constructor PandasFeaturesDecoder(features: Optional[Features]) - changed:
constructor PythonFeaturesDecoder(features) → constructor PythonFeaturesDecoder(features: Optional[Features]) - changed:
method PythonFormatter.format_batch(pa_table: pa.Table) -> dict → method PythonFormatter.format_batch(pa_table: pa.Table) -> Mapping - changed:
method PythonFormatter.format_row(pa_table: pa.Table) -> dict → method PythonFormatter.format_row(pa_table: pa.Table) -> Mapping - added:
class LazyBatch(LazyDict) - added:
class LazyDict(MutableMapping) - added:
class LazyRow(LazyDict) - added:
class TensorFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]) - added:
constructor LazyDict(pa_table: pa.Table, formatter: Formatter) - added:
constructor PythonFormatter(features=…, lazy=…) - added:
method LazyBatch.format(key) - added:
method LazyDict.copy() - added:
method LazyDict.format(key) - added:
method LazyRow.format(key) - added:
method TensorFormatter.recursive_tensorize(data_struct: dict) - added:
static method LazyDict.fromkeys(iterable, value=…)
- changed:
class JaxFormatter(Formatter[dict, 'jnp.ndarray', dict]) → class JaxFormatter(TensorFormatter[Mapping, 'jax.Array', Mapping]) - changed:
constructor JaxFormatter(features=…, decoded=…, **jnp_array_kwargs) → constructor JaxFormatter(features=…, device=…, **jnp_array_kwargs) - changed:
method JaxFormatter.format_batch(pa_table: pa.Table) -> dict → method JaxFormatter.format_batch(pa_table: pa.Table) -> Mapping - changed:
method JaxFormatter.format_column(pa_table: pa.Table) -> jnp.ndarray → method JaxFormatter.format_column(pa_table: pa.Table) -> jax.Array - changed:
method JaxFormatter.format_row(pa_table: pa.Table) -> dict → method JaxFormatter.format_row(pa_table: pa.Table) -> Mapping - added:
variable DEVICE_MAPPING: Optional[dict] - added:
variable logger
- added:
class NumpyFormatter(TensorFormatter[Mapping, np.ndarray, Mapping]) - added:
constructor NumpyFormatter(features=…, **np_array_kwargs) - added:
method NumpyFormatter.format_batch(pa_table: pa.Table) -> Mapping - added:
method NumpyFormatter.format_column(pa_table: pa.Table) -> np.ndarray - added:
method NumpyFormatter.format_row(pa_table: pa.Table) -> Mapping - added:
method NumpyFormatter.recursive_tensorize(data_struct: dict)
- added:
class PolarsArrowExtractor(BaseArrowExtractor['pl.DataFrame', 'pl.Series', 'pl.DataFrame']) - added:
class PolarsFeaturesDecoder - added:
class PolarsFormatter(TensorFormatter[Mapping, 'pl.DataFrame', Mapping]) - added:
constructor PolarsFeaturesDecoder(features: Optional[Features]) - added:
constructor PolarsFormatter(features=…, **np_array_kwargs) - added:
method PolarsArrowExtractor.extract_batch(pa_table: pa.Table) -> pl.DataFrame - added:
method PolarsArrowExtractor.extract_column(pa_table: pa.Table) -> pl.Series - added:
method PolarsArrowExtractor.extract_row(pa_table: pa.Table) -> pl.DataFrame - added:
method PolarsFeaturesDecoder.decode_batch(batch: pl.DataFrame) -> pl.DataFrame - added:
method PolarsFeaturesDecoder.decode_column(column: pl.Series, column_name: str) -> pl.Series - added:
method PolarsFeaturesDecoder.decode_row(row: pl.DataFrame) -> pl.DataFrame - added:
method PolarsFormatter.format_batch(pa_table: pa.Table) -> pl.DataFrame - added:
method PolarsFormatter.format_column(pa_table: pa.Table) -> pl.Series - added:
method PolarsFormatter.format_row(pa_table: pa.Table) -> pl.DataFrame
- changed:
class TFFormatter(Formatter[dict, 'tf.Tensor', dict]) → class TFFormatter(TensorFormatter[Mapping, 'tf.Tensor', Mapping]) - changed:
constructor TFFormatter(features=…, decoded=…, **tf_tensor_kwargs) → constructor TFFormatter(features=…, **tf_tensor_kwargs) - changed:
method TFFormatter.format_batch(pa_table: pa.Table) -> dict → method TFFormatter.format_batch(pa_table: pa.Table) -> Mapping - changed:
method TFFormatter.format_row(pa_table: pa.Table) -> dict → method TFFormatter.format_row(pa_table: pa.Table) -> Mapping
- changed:
class TorchFormatter(Formatter[dict, 'torch.Tensor', dict]) → class TorchFormatter(TensorFormatter[Mapping, 'torch.Tensor', Mapping]) - changed:
constructor TorchFormatter(features=…, decoded=…, **torch_tensor_kwargs) → constructor TorchFormatter(features=…, **torch_tensor_kwargs) - changed:
method TorchFormatter.format_batch(pa_table: pa.Table) -> dict → method TorchFormatter.format_batch(pa_table: pa.Table) -> Mapping - changed:
method TorchFormatter.format_row(pa_table: pa.Table) -> dict → method TorchFormatter.format_row(pa_table: pa.Table) -> Mapping
- added:
function convert_to_parquet(repo_id: str, revision: Optional[str]=…, token: Optional[Union[bool, str]]=…, trust_remote_code: Optional[bool]=…) -> CommitInfo - added:
function delete_from_hub(repo_id: str, config_name: str, revision: Optional[str]=…, token: Optional[Union[bool, str]]=…) -> CommitInfo
- changed:
method DatasetInfo.write_to_directory(dataset_info_dir) → method DatasetInfo.write_to_directory(dataset_info_dir, pretty_print=…, fs=…, storage_options: Optional[dict]=…) - changed:
method DatasetInfosDict.write_to_directory(dataset_infos_dir, overwrite=…) → method DatasetInfosDict.write_to_directory(dataset_infos_dir, overwrite=…, pretty_print=…) -> None - changed:
method MetricInfo.write_to_directory(metric_info_dir) → method MetricInfo.write_to_directory(metric_info_dir, pretty_print=…) - changed:
static method DatasetInfo.from_directory(dataset_info_dir: str) -> DatasetInfo → static method DatasetInfo.from_directory(dataset_info_dir: str, fs=…, storage_options: Optional[dict]=…) -> DatasetInfo - changed:
static method DatasetInfosDict.from_directory(dataset_infos_dir) → static method DatasetInfosDict.from_directory(dataset_infos_dir) -> DatasetInfosDict - added:
field DatasetInfo.dataset_name: Optional[str] - added:
method DatasetInfosDict.to_dataset_card_data(dataset_card_data: DatasetCardData) -> None - added:
static method DatasetInfosDict.from_dataset_card_data(dataset_card_data: DatasetCardData) -> DatasetInfosDict
- changed:
function get_dataset_config_names(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, force_local_path: Optional[str]=…, dynamic_modules_path: Optional[str]=…, data_files: Optional[Union[Dict, List, str]]=…, **download_kwargs) → function get_dataset_config_names(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, data_files: Optional[Union[Dict, List, str]]=…, **download_kwargs) - changed:
function get_dataset_infos(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, force_local_path: Optional[str]=…, dynamic_modules_path: Optional[str]=…, data_files: Optional[Union[Dict, List, str]]=…, **download_kwargs) → function get_dataset_infos(path: str, data_files: Optional[Union[Dict, List, str]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, revision: Optional[Union[str, Version]]=…, token: Optional[Union[bool, str]]=…, use_auth_token=…, **config_kwargs) - changed:
function get_dataset_split_names(path: str, config_name: Optional[str]=…, data_dir: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, cache_dir: Optional[str]=…, features: Optional[Features]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, revision: Optional[Union[str, Version]]=…, use_auth_token: O… → function get_dataset_split_names(path: str, config_name: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, revision: Optional[Union[str, Version]]=…, token: Optional[Union[bool, str]]=…, use_auth_token=…, **config_kwargs) - added:
function get_dataset_config_info(path: str, config_name: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, revision: Optional[Union[str, Version]]=…, token: Optional[Union[bool, str]]=…, use_auth_token=…, **config_kwargs) -> DatasetInfo - added:
function get_dataset_default_config_name(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, data_files: Optional[Union[Dict, List, str]]=…, **download_kwargs) -> Optional[str]
- changed:
constructor AbstractDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) → constructor AbstractDatasetReader(path_or_paths: Optional[NestedDataStructureLike[PathLike]]=…, split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, num_proc: Optional[int]=…, **kwargs) - changed:
method AbstractDatasetReader.read() -> Union[Dataset, DatasetDict] → method AbstractDatasetReader.read() -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict] - added:
class AbstractDatasetInputStream(ABC) - added:
constructor AbstractDatasetInputStream(features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, num_proc: Optional[int]=…, **kwargs) - added:
method AbstractDatasetInputStream.read() -> Union[Dataset, IterableDataset]
- changed:
constructor CsvDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) → constructor CsvDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, num_proc: Optional[int]=…, **kwargs) - changed:
constructor CsvDatasetWriter(dataset: Dataset, path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, **to_csv_kwargs) → constructor CsvDatasetWriter(dataset: Dataset, path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, storage_options: Optional[dict]=…, **to_csv_kwargs)
- added:
class GeneratorDatasetInputStream(AbstractDatasetInputStream) - added:
constructor GeneratorDatasetInputStream(generator: Callable, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, gen_kwargs: Optional[dict]=…, num_proc: Optional[int]=…, split: NamedSplit=…, **kwargs) - added:
method GeneratorDatasetInputStream.read()
- removed:
class IOHandler - removed:
constructor IOHandler(file_name, file_mode, compression)
- changed:
constructor JsonDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, field: Optional[str]=…, **kwargs) → constructor JsonDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, field: Optional[str]=…, num_proc: Optional[int]=…, **kwargs) - changed:
constructor JsonDatasetWriter(dataset: Dataset, path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, **to_json_kwargs) → constructor JsonDatasetWriter(dataset: Dataset, path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, num_proc: Optional[int]=…, storage_options: Optional[dict]=…, **to_json_kwargs)
- changed:
constructor ParquetDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) → constructor ParquetDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, num_proc: Optional[int]=…, **kwargs) - changed:
constructor ParquetDatasetWriter(dataset: Dataset, path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, **parquet_writer_kwargs) → constructor ParquetDatasetWriter(dataset: Dataset, path_or_buf: Union[PathLike, BinaryIO], batch_size: Optional[int]=…, storage_options: Optional[dict]=…, **parquet_writer_kwargs) - added:
function get_writer_batch_size(features: Features) -> Optional[int]
- added:
class SparkDatasetReader(AbstractDatasetReader) - added:
constructor SparkDatasetReader(df: pyspark.sql.DataFrame, split: Optional[NamedSplit]=…, features: Optional[Features]=…, streaming: bool=…, cache_dir: str=…, keep_in_memory: bool=…, working_dir: str=…, load_from_cache_file: bool=…, file_format: str=…, **kwargs) - added:
method SparkDatasetReader.read()
- added:
class SqlDatasetReader(AbstractDatasetInputStream) - added:
class SqlDatasetWriter - added:
constructor SqlDatasetReader(sql: Union[str, sqlalchemy.sql.Selectable], con: Union[str, sqlalchemy.engine.Connection, sqlalchemy.engine.Engine, sqlite3.Connection], features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) - added:
constructor SqlDatasetWriter(dataset: Dataset, name: str, con: Union[str, sqlalchemy.engine.Connection, sqlalchemy.engine.Engine, sqlite3.Connection], batch_size: Optional[int]=…, num_proc: Optional[int]=…, **to_sql_kwargs) - added:
method SqlDatasetReader.read() - added:
method SqlDatasetWriter.write() -> int
- changed:
constructor TextDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, **kwargs) → constructor TextDatasetReader(path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit]=…, features: Optional[Features]=…, cache_dir: str=…, keep_in_memory: bool=…, streaming: bool=…, num_proc: Optional[int]=…, **kwargs)
- removed:
class ShardShuffledExamplesIterable(ExamplesIterable) - removed:
constructor ShardShuffledExamplesIterable(generate_examples_fn: Callable, kwargs: dict, seed: Optional[int]) - removed:
field ShufflingConfig.seed: Optional[int] - removed:
function iterable_dataset(ex_iterable: Iterable, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…, format_type: Optional[str]=…, shuffling: Optional[ShufflingConfig]=…) - changed:
constructor BufferShuffledExamplesIterable(ex_iterable: _BaseExamplesIterable, buffer_size: int, seed: Optional[int]) → constructor BufferShuffledExamplesIterable(ex_iterable: _BaseExamplesIterable, buffer_size: int, generator: np.random.Generator) - changed:
constructor CyclingMultiSourcesExamplesIterable(ex_iterables: List[_BaseExamplesIterable]) → constructor CyclingMultiSourcesExamplesIterable(ex_iterables: List[_BaseExamplesIterable], stopping_strategy: Literal['first_exhausted', 'all_exhausted']=…) - changed:
constructor ExamplesIterable(generate_examples_fn: Callable, kwargs: dict) → constructor ExamplesIterable(generate_examples_fn: Callable[..., Tuple[Key, dict]], kwargs: dict) - changed:
constructor IterableDataset(ex_iterable: _BaseExamplesIterable, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…, format_type: Optional[str]=…, shuffling: Optional[ShufflingConfig]=…) → constructor IterableDataset(ex_iterable: _BaseExamplesIterable, info: Optional[DatasetInfo]=…, split: Optional[NamedSplit]=…, formatting: Optional[FormattingConfig]=…, shuffling: Optional[ShufflingConfig]=…, distributed: Optional[DistributedConfig]=…, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]]=…, format_type=…) - changed:
constructor MappedExamplesIterable(ex_iterable: _BaseExamplesIterable, function: Callable, batched: bool=…, batch_size: int=…) → constructor MappedExamplesIterable(ex_iterable: _BaseExamplesIterable, function: Callable, with_indices: bool=…, input_columns: Optional[List[str]]=…, batched: bool=…, batch_size: Optional[int]=…, drop_last_batch: bool=…, remove_columns: Optional[List[str]]=…, fn_kwargs: Optional[dict]=…, formatting: Optional[FormattingConfig]=…, format_type=…) - changed:
constructor RandomlyCyclingMultiSourcesExamplesIterable(ex_iterables, seed: Optional[int]=…, probabilities: Optional[List[float]]=…) → constructor RandomlyCyclingMultiSourcesExamplesIterable(ex_iterables: List[_BaseExamplesIterable], generator: np.random.Generator, probabilities: Optional[List[float]]=…, stopping_strategy: Literal['first_exhausted', 'all_exhausted']=…) - changed:
constructor SkipExamplesIterable(ex_iterable: _BaseExamplesIterable, n: int) → constructor SkipExamplesIterable(ex_iterable: _BaseExamplesIterable, n: int, block_sources_order_when_shuffling: bool=…, split_when_sharding: bool=…) - changed:
constructor TakeExamplesIterable(ex_iterable: _BaseExamplesIterable, n: int) → constructor TakeExamplesIterable(ex_iterable: _BaseExamplesIterable, n: int, block_sources_order_when_shuffling: bool=…, split_when_sharding: bool=…) - changed:
constructor TypedExamplesIterable(ex_iterable: _BaseExamplesIterable, features: Features) → constructor TypedExamplesIterable(ex_iterable: _BaseExamplesIterable, features: Features, token_per_repo_id: Dict[str, Union[str, bool, None]]) - changed:
method BufferShuffledExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> BufferShuffledExamplesIterable → method BufferShuffledExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> BufferShuffledExamplesIterable - changed:
method CyclingMultiSourcesExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> CyclingMultiSourcesExamplesIterable → method CyclingMultiSourcesExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> CyclingMultiSourcesExamplesIterable - changed:
method ExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> ExamplesIterable → method ExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> ExamplesIterable - changed:
method IterableDataset.map(function: Callable, batched: bool=…, batch_size: int=…) → method IterableDataset.map(function: Optional[Callable]=…, with_indices: bool=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, drop_last_batch: bool=…, remove_columns: Optional[Union[str, List[str]]]=…, features: Optional[Features]=…, fn_kwargs: Optional[dict]=…) -> IterableDataset - changed:
method IterableDataset.shuffle(buffer_size, seed=…) -> IterableDataset → method IterableDataset.shuffle(seed=…, generator: Optional[np.random.Generator]=…, buffer_size: int=…) -> IterableDataset - changed:
method IterableDataset.skip(n) -> IterableDataset → method IterableDataset.skip(n: int) -> IterableDataset - changed:
method IterableDataset.take(n) -> IterableDataset → method IterableDataset.take(n: int) -> IterableDataset - changed:
method MappedExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> MappedExamplesIterable → method MappedExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> MappedExamplesIterable - changed:
method RandomlyCyclingMultiSourcesExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> RandomlyCyclingMultiSourcesExamplesIterable → method RandomlyCyclingMultiSourcesExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> RandomlyCyclingMultiSourcesExamplesIterable - changed:
method SkipExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> SkipExamplesIterable → method SkipExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> SkipExamplesIterable - changed:
method TakeExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> TakeExamplesIterable → method TakeExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> TakeExamplesIterable - changed:
method TypedExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> TypedExamplesIterable → method TypedExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> TypedExamplesIterable - added:
class ArrowExamplesIterable(_BaseExamplesIterable) - added:
class DistributedConfig - added:
class FilteredExamplesIterable(_BaseExamplesIterable) - added:
class FormattingConfig - added:
class HorizontallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable) - added:
class RebatchedArrowExamplesIterable(_BaseExamplesIterable) - added:
class SelectColumnsIterable(_BaseExamplesIterable) - added:
class ShuffledDataSourcesArrowExamplesIterable(ArrowExamplesIterable) - added:
class ShuffledDataSourcesExamplesIterable(ExamplesIterable) - added:
class StepExamplesIterable(_BaseExamplesIterable) - added:
class VerticallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable) - added:
constructor ArrowExamplesIterable(generate_tables_fn: Callable[..., Tuple[Key, pa.Table]], kwargs: dict) - added:
constructor FilteredExamplesIterable(ex_iterable: _BaseExamplesIterable, function: Callable, with_indices: bool=…, input_columns: Optional[List[str]]=…, batched: bool=…, batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…, formatting: Optional[FormattingConfig]=…, format_type=…) - added:
constructor HorizontallyConcatenatedMultiSourcesExamplesIterable(ex_iterables: List[_BaseExamplesIterable]) - added:
constructor RebatchedArrowExamplesIterable(ex_iterable: _BaseExamplesIterable, batch_size: Optional[int], drop_last_batch: bool=…) - added:
constructor SelectColumnsIterable(ex_iterable: _BaseExamplesIterable, column_names: List[str]) - added:
constructor ShuffledDataSourcesArrowExamplesIterable(generate_tables_fn: Callable[..., Tuple[Key, pa.Table]], kwargs: dict, generator: np.random.Generator) - added:
constructor ShuffledDataSourcesExamplesIterable(generate_examples_fn: Callable[..., Tuple[Key, dict]], kwargs: dict, generator: np.random.Generator) - added:
constructor StepExamplesIterable(ex_iterable: _BaseExamplesIterable, step: int, offset: int) - added:
constructor VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables: List[_BaseExamplesIterable]) - added:
field DistributedConfig.rank: int - added:
field DistributedConfig.world_size: int - added:
field FormattingConfig.format_type: Optional[str] - added:
field ShufflingConfig.generator: np.random.Generator - added:
function add_column_fn(example: Dict, idx: int, name: str, column: List[Dict]) - added:
function identity_func(x) - added:
method ArrowExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> ArrowExamplesIterable - added:
method ArrowExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> ArrowExamplesIterable - added:
method BufferShuffledExamplesIterable.load_state_dict(state_dict: dict) -> dict - added:
method BufferShuffledExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> BufferShuffledExamplesIterable - added:
method CyclingMultiSourcesExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> CyclingMultiSourcesExamplesIterable - added:
method ExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> ExamplesIterable - added:
method FilteredExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> FilteredExamplesIterable - added:
method FilteredExamplesIterable.shuffle_data_sources(seed: Optional[int]) -> FilteredExamplesIterable - added:
method HorizontallyConcatenatedMultiSourcesExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> HorizontallyConcatenatedMultiSourcesExamplesIterable - added:
method HorizontallyConcatenatedMultiSourcesExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> HorizontallyConcatenatedMultiSourcesExamplesIterable - added:
method IterableDataset.add_column(name: str, column: Union[list, np.array]) -> IterableDataset - added:
method IterableDataset.batch(batch_size: int, drop_last_batch: bool=…) -> IterableDataset - added:
method IterableDataset.cast(features: Features) -> IterableDataset - added:
method IterableDataset.filter(function: Optional[Callable]=…, with_indices=…, input_columns: Optional[Union[str, List[str]]]=…, batched: bool=…, batch_size: Optional[int]=…, fn_kwargs: Optional[dict]=…) -> IterableDataset - added:
method IterableDataset.iter(batch_size: int, drop_last_batch: bool=…) - added:
method IterableDataset.load_state_dict(state_dict: dict) -> None - added:
method IterableDataset.rename_column(original_column_name: str, new_column_name: str) -> IterableDataset - added:
method IterableDataset.rename_columns(column_mapping: Dict[str, str]) -> IterableDataset - added:
method IterableDataset.select_columns(column_names: Union[str, List[str]]) -> IterableDataset - added:
method IterableDataset.state_dict() -> dict - added:
method MappedExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> MappedExamplesIterable - added:
method RandomlyCyclingMultiSourcesExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> RandomlyCyclingMultiSourcesExamplesIterable - added:
method RebatchedArrowExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> RebatchedArrowExamplesIterable - added:
method RebatchedArrowExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> RebatchedArrowExamplesIterable - added:
method SelectColumnsIterable.shard_data_sources(worker_id: int, num_workers: int) -> SelectColumnsIterable - added:
method SelectColumnsIterable.shuffle_data_sources(generator: np.random.Generator) -> SelectColumnsIterable - added:
method ShuffledDataSourcesArrowExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> ArrowExamplesIterable - added:
method ShuffledDataSourcesExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> ExamplesIterable - added:
method SkipExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> SkipExamplesIterable - added:
method StepExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> StepExamplesIterable - added:
method StepExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> StepExamplesIterable - added:
method TakeExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> TakeExamplesIterable - added:
method TypedExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> TypedExamplesIterable - added:
method VerticallyConcatenatedMultiSourcesExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> VerticallyConcatenatedMultiSourcesExamplesIterable - added:
method VerticallyConcatenatedMultiSourcesExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> VerticallyConcatenatedMultiSourcesExamplesIterable - added:
property ArrowExamplesIterable.iter_arrow - added:
property ArrowExamplesIterable.n_shards -> int - added:
property FilteredExamplesIterable.iter_arrow - added:
property FilteredExamplesIterable.n_shards -> int - added:
property HorizontallyConcatenatedMultiSourcesExamplesIterable.n_shards -> int - added:
property IterableDataset.column_names -> Optional[List[str]] - added:
property IterableDataset.epoch -> int - added:
property MappedExamplesIterable.iter_arrow - added:
property RebatchedArrowExamplesIterable.iter_arrow - added:
property RebatchedArrowExamplesIterable.n_shards -> int - added:
property SelectColumnsIterable.iter_arrow - added:
property SelectColumnsIterable.n_shards -> int - added:
property StepExamplesIterable.n_shards -> int - added:
property TypedExamplesIterable.iter_arrow - added:
property VerticallyConcatenatedMultiSourcesExamplesIterable.iter_arrow - added:
property VerticallyConcatenatedMultiSourcesExamplesIterable.n_shards -> int - added:
static method IterableDataset.from_file(filename: str) -> IterableDataset - added:
static method IterableDataset.from_generator(generator: Callable, features: Optional[Features]=…, gen_kwargs: Optional[dict]=…, split: NamedSplit=…) -> IterableDataset - added:
static method IterableDataset.from_spark(df: pyspark.sql.DataFrame, split: Optional[NamedSplit]=…, features: Optional[Features]=…, **kwargs) -> IterableDataset - added:
static method SkipExamplesIterable.split_number(num, n) - added:
static method TakeExamplesIterable.split_number(num, n) - added:
variable Key - added:
variable logger
- changed:
constructor DuplicatedKeysError(key) → constructor DuplicatedKeysError(key, duplicate_key_indices, fix_msg=…)
- removed:
class CanonicalDatasetModuleFactory(_DatasetModuleFactory) - removed:
class CanonicalMetricModuleFactory(_MetricModuleFactory) - removed:
class CommunityDatasetModuleFactoryWithScript(_DatasetModuleFactory) - removed:
class CommunityDatasetModuleFactoryWithoutScript(_DatasetModuleFactory) - removed:
constructor CanonicalDatasetModuleFactory(name: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, dynamic_modules_path: Optional[str]=…) - removed:
constructor CanonicalMetricModuleFactory(name: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, dynamic_modules_path: Optional[str]=…) - removed:
constructor CommunityDatasetModuleFactoryWithScript(name: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, dynamic_modules_path: Optional[str]=…) - removed:
constructor CommunityDatasetModuleFactoryWithoutScript(name: str, revision: Optional[Union[str, Version]]=…, data_files: Optional[Union[str, List, Dict]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…) - removed:
function convert_github_url(url_path: str) -> Tuple[str, Optional[str]] - removed:
function extend_dataset_builder_for_streaming(builder: DatasetBuilder, use_auth_token: Optional[Union[bool, str]]=…) - removed:
function get_imports(file_path: str) -> Tuple[str, str, str, str] - removed:
function infer_module_for_data_files_in_archives(data_files_list: DataFilesList, use_auth_token: Optional[Union[bool, str]]) -> Optional[str] - removed:
function prepare_module(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, dataset: bool=…, force_local_path: Optional[str]=…, dynamic_modules_path: Optional[str]=…, data_files: Optional[Union[Dict, List, str]]=…, script_version=…, **download_kwargs) -> Union[Tuple[str, str], Tuple[str, str, Optional[str]]] - removed:
method CanonicalDatasetModuleFactory.download_dataset_infos_file(revision: Optional[str]) -> str - removed:
method CanonicalDatasetModuleFactory.download_loading_script(revision: Optional[str]) -> str - removed:
method CanonicalDatasetModuleFactory.get_module() -> DatasetModule - removed:
method CanonicalMetricModuleFactory.download_loading_script(revision: Optional[str]) -> str - removed:
method CanonicalMetricModuleFactory.get_module() -> MetricModule - removed:
method CommunityDatasetModuleFactoryWithScript.download_dataset_infos_file() -> str - removed:
method CommunityDatasetModuleFactoryWithScript.download_loading_script() -> str - removed:
method CommunityDatasetModuleFactoryWithScript.get_module() -> DatasetModule - removed:
method CommunityDatasetModuleFactoryWithoutScript.get_module() -> DatasetModule - removed:
variable DEFAULT_SPLIT - changed:
constructor CachedDatasetModuleFactory(name: str, dynamic_modules_path: Optional[str]=…) → constructor CachedDatasetModuleFactory(name: str, cache_dir: Optional[str]=…, dynamic_modules_path: Optional[str]=…) - changed:
constructor LocalDatasetModuleFactoryWithScript(path: str, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, dynamic_modules_path: Optional[str]=…) → constructor LocalDatasetModuleFactoryWithScript(path: str, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, trust_remote_code: Optional[bool]=…) - changed:
constructor LocalDatasetModuleFactoryWithoutScript(path: str, data_files: Optional[Union[str, List, Dict]]=…, download_mode: Optional[GenerateMode]=…) → constructor LocalDatasetModuleFactoryWithoutScript(path: str, data_dir: Optional[str]=…, data_files: Optional[Union[str, List, Dict]]=…, download_mode: Optional[Union[DownloadMode, str]]=…) - changed:
constructor LocalMetricModuleFactory(path: str, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, dynamic_modules_path: Optional[str]=…) → constructor LocalMetricModuleFactory(path: str, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, trust_remote_code: Optional[str]=…) - changed:
constructor PackagedDatasetModuleFactory(name: str, data_files: Optional[Union[str, List, Dict]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…) → constructor PackagedDatasetModuleFactory(name: str, data_dir: Optional[str]=…, data_files: Optional[Union[str, List, Dict]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…) - changed:
function dataset_module_factory(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, force_local_path: Optional[str]=…, dynamic_modules_path: Optional[str]=…, data_files: Optional[Union[Dict, List, str, DataFilesDict]]=…, **download_kwargs) -> DatasetModule → function dataset_module_factory(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, data_dir: Optional[str]=…, data_files: Optional[Union[Dict, List, str, DataFilesDict]]=…, cache_dir: Optional[str]=…, trust_remote_code: Optional[bool]=…, _require_default_conf… - changed:
function infer_module_for_data_files(data_files_list: DataFilesList, use_auth_token: Optional[Union[bool, str]]=…) -> Optional[str] → function infer_module_for_data_files(data_files: DataFilesDict, path: Optional[str]=…, download_config: Optional[DownloadConfig]=…) -> Tuple[Optional[str], Dict[str, Any]] - changed:
function load_dataset(path: str, name: Optional[str]=…, data_dir: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, split: Optional[Union[str, Split]]=…, cache_dir: Optional[str]=…, features: Optional[Features]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, ignore_verifications: bool=…, keep_in_me… → function load_dataset(path: str, name: Optional[str]=…, data_dir: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, split: Optional[Union[str, Split]]=…, cache_dir: Optional[str]=…, features: Optional[Features]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, verification_mode: Optional[… - changed:
function load_dataset_builder(path: str, name: Optional[str]=…, data_dir: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, cache_dir: Optional[str]=…, features: Optional[Features]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, revision: Optional[Union[str, Version]]=…, use_auth_token: Optional[Un… → function load_dataset_builder(path: str, name: Optional[str]=…, data_dir: Optional[str]=…, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]]=…, cache_dir: Optional[str]=…, features: Optional[Features]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, revision: Optional[Union[str, Version]]=…, token: Optional… - changed:
function load_from_disk(dataset_path: str, fs=…, keep_in_memory: Optional[bool]=…) -> Union[Dataset, DatasetDict] → function load_from_disk(dataset_path: PathLike, fs=…, keep_in_memory: Optional[bool]=…, storage_options: Optional[dict]=…) -> Union[Dataset, DatasetDict] - changed:
function load_metric(path: str, config_name: Optional[str]=…, process_id: int=…, num_process: int=…, cache_dir: Optional[str]=…, experiment_id: Optional[str]=…, keep_in_memory: bool=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, revision: Optional[Union[str, Version]]=…, script_version=…, **metric_init_kwargs) -> Metric → function load_metric(path: str, config_name: Optional[str]=…, process_id: int=…, num_process: int=…, cache_dir: Optional[str]=…, experiment_id: Optional[str]=…, keep_in_memory: bool=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, revision: Optional[Union[str, Version]]=…, trust_remote_code: Optional[bool]=…, **metric_init_kwargs) -> Metric - changed:
function metric_module_factory(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[GenerateMode]=…, force_local_path: Optional[str]=…, dynamic_modules_path: Optional[str]=…, **download_kwargs) -> MetricModule → function metric_module_factory(path: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, trust_remote_code: Optional[bool]=…, **download_kwargs) -> MetricModule - added:
class BuilderConfigsParameters - added:
class GithubMetricModuleFactory(_MetricModuleFactory) - added:
class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory) - added:
class HubDatasetModuleFactoryWithScript(_DatasetModuleFactory) - added:
class HubDatasetModuleFactoryWithoutScript(_DatasetModuleFactory) - added:
constructor GithubMetricModuleFactory(name: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, trust_remote_code: Optional[str]=…) - added:
constructor HubDatasetModuleFactoryWithParquetExport(name: str, revision: Optional[str]=…, download_config: Optional[DownloadConfig]=…) - added:
constructor HubDatasetModuleFactoryWithScript(name: str, revision: Optional[Union[str, Version]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…, dynamic_modules_path: Optional[str]=…, trust_remote_code: Optional[bool]=…) - added:
constructor HubDatasetModuleFactoryWithoutScript(name: str, revision: Optional[Union[str, Version]]=…, data_dir: Optional[str]=…, data_files: Optional[Union[str, List, Dict]]=…, download_config: Optional[DownloadConfig]=…, download_mode: Optional[Union[DownloadMode, str]]=…) - added:
field BuilderConfigsParameters.builder_configs: Optional[List[BuilderConfig]] - added:
field BuilderConfigsParameters.default_config_name: Optional[str] - added:
field BuilderConfigsParameters.metadata_configs: Optional[MetadataConfigs] - added:
field DatasetModule.builder_configs_parameters: BuilderConfigsParameters - added:
field DatasetModule.dataset_infos: Optional[DatasetInfosDict] - added:
field DatasetModule.importable_file_path: Optional[str] - added:
function configure_builder_class(builder_cls: Type[DatasetBuilder], builder_configs: List[BuilderConfig], default_config_name: Optional[str], dataset_name: str) -> Type[DatasetBuilder] - added:
function create_builder_configs_from_metadata_configs(module_path: str, metadata_configs: MetadataConfigs, supports_metadata: bool, base_path: Optional[str]=…, default_builder_kwargs: Dict[str, Any]=…, download_config: Optional[DownloadConfig]=…) -> Tuple[List[BuilderConfig], str] - added:
function get_dataset_builder_class(dataset_module: DatasetModule, dataset_name: Optional[str]=…) -> Type[DatasetBuilder] - added:
function infer_module_for_data_files_list(data_files_list: DataFilesList, download_config: Optional[DownloadConfig]=…) -> Tuple[Optional[str], dict] - added:
function infer_module_for_data_files_list_in_archives(data_files_list: DataFilesList, download_config: Optional[DownloadConfig]=…) -> Tuple[Optional[str], dict] - added:
function resolve_trust_remote_code(trust_remote_code: Optional[bool], repo_id: str) -> bool - added:
method GithubMetricModuleFactory.download_loading_script(revision: Optional[str]) -> str - added:
method GithubMetricModuleFactory.get_module() -> MetricModule - added:
method HubDatasetModuleFactoryWithParquetExport.get_module() -> DatasetModule - added:
method HubDatasetModuleFactoryWithScript.download_dataset_infos_file() -> str - added:
method HubDatasetModuleFactoryWithScript.download_dataset_readme_file() -> str - added:
method HubDatasetModuleFactoryWithScript.download_loading_script() -> str - added:
method HubDatasetModuleFactoryWithScript.get_module() -> DatasetModule - added:
method HubDatasetModuleFactoryWithoutScript.get_module() -> DatasetModule
- changed:
method Metric.add(*, prediction=…, reference=…) → method Metric.add(*, prediction=…, reference=…, **kwargs) - changed:
method Metric.add_batch(*, predictions=…, references=…) → method Metric.add_batch(*, predictions=…, references=…, **kwargs) - added:
function summarize_if_long_list(obj)
- removed:
function filename_for_dataset_split(dataset_name, split, filetype_suffix=…) - removed:
function filepath_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=…) - added:
function filenames_for_dataset_split(path, dataset_name, split, filetype_suffix=…, shard_lengths=…) - added:
variable INVALID_WINDOWS_CHARACTERS_IN_PATH
- removed:
function hash_python_lines(lines: List[str]) -> str - added:
export arrow - added:
export audiofolder - added:
export cache - added:
export imagefolder - added:
export sql - added:
export webdataset
- added:
class Arrow(datasets.ArrowBasedBuilder) - added:
class ArrowConfig(datasets.BuilderConfig) - added:
field Arrow.BUILDER_CONFIG_CLASS - added:
field ArrowConfig.features: Optional[datasets.Features] - added:
variable logger
- added:
class AudioFolder(folder_based_builder.FolderBasedBuilder) - added:
class AudioFolderConfig(folder_based_builder.FolderBasedBuilderConfig) - added:
field AudioFolder.BASE_COLUMN_NAME - added:
field AudioFolder.BASE_FEATURE - added:
field AudioFolder.BUILDER_CONFIG_CLASS - added:
field AudioFolder.CLASSIFICATION_TASK - added:
field AudioFolder.EXTENSIONS: List[str] - added:
field AudioFolderConfig.drop_labels: bool - added:
field AudioFolderConfig.drop_metadata: bool - added:
variable AUDIO_EXTENSIONS - added:
variable logger
- added:
class Cache(datasets.ArrowBasedBuilder) - added:
constructor Cache(cache_dir: Optional[str]=…, dataset_name: Optional[str]=…, config_name: Optional[str]=…, version: Optional[str]=…, hash: Optional[str]=…, base_path: Optional[str]=…, info: Optional[datasets.DatasetInfo]=…, features: Optional[datasets.Features]=…, token: Optional[Union[bool, str]]=…, use_auth_token=…, repo_id: Optional[str]=…, data_files: Optional[Union[str, list, dict, datasets.… - added:
method Cache.download_and_prepare(output_dir: Optional[str]=…, *args, **kwargs) - added:
variable logger
- removed:
property CsvConfig.read_csv_kwargs - changed:
field CsvConfig.engine: Optional[str] → field CsvConfig.engine: Optional[Literal['c', 'python', 'pyarrow']] - added:
field CsvConfig.converters: Dict[Union[int, str], Callable[[Any], Any]] - added:
field CsvConfig.date_format: Optional[str] - added:
property CsvConfig.pd_read_csv_kwargs
- added:
class FolderBasedBuilder(datasets.GeneratorBasedBuilder) - added:
class FolderBasedBuilderConfig(datasets.BuilderConfig) - added:
field FolderBasedBuilder.BASE_COLUMN_NAME: str - added:
field FolderBasedBuilder.BASE_FEATURE: Type[FeatureType] - added:
field FolderBasedBuilder.BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig - added:
field FolderBasedBuilder.CLASSIFICATION_TASK: TaskTemplate - added:
field FolderBasedBuilder.EXTENSIONS: List[str] - added:
field FolderBasedBuilder.METADATA_FILENAMES: List[str] - added:
field FolderBasedBuilderConfig.drop_labels: bool - added:
field FolderBasedBuilderConfig.drop_metadata: bool - added:
field FolderBasedBuilderConfig.features: Optional[datasets.Features] - added:
function count_path_segments(path) - added:
variable logger
- added:
class Generator(datasets.GeneratorBasedBuilder) - added:
class GeneratorConfig(datasets.BuilderConfig) - added:
field Generator.BUILDER_CONFIG_CLASS - added:
field GeneratorConfig.features: Optional[datasets.Features] - added:
field GeneratorConfig.gen_kwargs: Optional[dict] - added:
field GeneratorConfig.generator: Optional[Callable] - added:
field GeneratorConfig.split: datasets.NamedSplit
- added:
class ImageFolder(folder_based_builder.FolderBasedBuilder) - added:
class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig) - added:
field ImageFolder.BASE_COLUMN_NAME - added:
field ImageFolder.BASE_FEATURE - added:
field ImageFolder.BUILDER_CONFIG_CLASS - added:
field ImageFolder.CLASSIFICATION_TASK - added:
field ImageFolder.EXTENSIONS: List[str] - added:
field ImageFolderConfig.drop_labels: bool - added:
field ImageFolderConfig.drop_metadata: bool - added:
variable IMAGE_EXTENSIONS - added:
variable logger
- removed:
property JsonConfig.schema - added:
field JsonConfig.encoding: str - added:
field JsonConfig.encoding_errors: Optional[str] - added:
function pandas_read_json(path_or_buf, **kwargs) - added:
function ujson_dumps(*args, **kwargs) - added:
function ujson_loads(*args, **kwargs)
- added:
class PandasConfig(datasets.BuilderConfig) - added:
field Pandas.BUILDER_CONFIG_CLASS - added:
field PandasConfig.features: Optional[datasets.Features]
- changed:
field ParquetConfig.batch_size: int → field ParquetConfig.batch_size: Optional[int]
- added:
class Spark(datasets.DatasetBuilder) - added:
class SparkConfig(datasets.BuilderConfig) - added:
class SparkExamplesIterable(_BaseExamplesIterable) - added:
constructor Spark(df: pyspark.sql.DataFrame, cache_dir: str=…, working_dir: str=…, **config_kwargs) - added:
constructor SparkExamplesIterable(df: pyspark.sql.DataFrame, partition_order=…) - added:
field Spark.BUILDER_CONFIG_CLASS - added:
field SparkConfig.features: Optional[datasets.Features] - added:
method SparkExamplesIterable.load_state_dict(state_dict: dict) -> dict - added:
method SparkExamplesIterable.shard_data_sources(worker_id: int, num_workers: int) -> SparkExamplesIterable - added:
method SparkExamplesIterable.shuffle_data_sources(generator: np.random.Generator) -> SparkExamplesIterable - added:
property SparkExamplesIterable.n_shards -> int - added:
variable logger
- added:
class Sql(datasets.ArrowBasedBuilder) - added:
class SqlConfig(datasets.BuilderConfig) - added:
field Sql.BUILDER_CONFIG_CLASS - added:
field SqlConfig.chunksize: Optional[int] - added:
field SqlConfig.coerce_float: bool - added:
field SqlConfig.columns: Optional[List[str]] - added:
field SqlConfig.con: Union[str, sqlalchemy.engine.Connection, sqlalchemy.engine.Engine, sqlite3.Connection] - added:
field SqlConfig.features: Optional[datasets.Features] - added:
field SqlConfig.index_col: Optional[Union[str, List[str]]] - added:
field SqlConfig.params: Optional[Union[List, Tuple, Dict]] - added:
field SqlConfig.parse_dates: Optional[Union[List, Dict]] - added:
field SqlConfig.sql: Union[str, sqlalchemy.sql.Selectable] - added:
method SqlConfig.create_config_id(config_kwargs: dict, custom_features: Optional[datasets.Features]=…) -> str - added:
property SqlConfig.pd_read_sql_kwargs - added:
variable logger
- added:
field TextConfig.encoding_errors: Optional[str] - added:
field TextConfig.errors: InitVar[Optional[str]]
- added:
class WebDataset(datasets.GeneratorBasedBuilder) - added:
field WebDataset.AUDIO_EXTENSIONS: List[str] - added:
field WebDataset.DECODERS: Dict[str, Callable[[Any], Any]] - added:
field WebDataset.DEFAULT_WRITER_BATCH_SIZE - added:
field WebDataset.IMAGE_EXTENSIONS: List[str] - added:
field WebDataset.NUM_EXAMPLES_FOR_FEATURES_INFERENCE - added:
function cbor_loads(data: bytes) - added:
function msgpack_loads(data: bytes) - added:
function npy_loads(data: bytes) - added:
function npz_loads(data: bytes) - added:
function tenbin_loads(data: bytes) - added:
function text_loads(data: bytes) - added:
function torch_loads(data: bytes) - added:
variable AUDIO_EXTENSIONS - added:
variable DECODERS - added:
variable IMAGE_EXTENSIONS - added:
variable logger
- added:
export ParallelBackendConfig - added:
export parallel_backend - added:
export parallel_map
- added:
class ParallelBackendConfig - added:
field ParallelBackendConfig.backend_name - added:
function parallel_backend(backend_name: str) - added:
function parallel_map(function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func) - added:
variable logger
- changed:
constructor FaissIndex(device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…) → constructor FaissIndex(device: Optional[Union[int, List[int]]]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…) - changed:
method BaseIndex.search(query, k: int=…) -> SearchResults → method BaseIndex.search(query, k: int=…, **kwargs) -> SearchResults - changed:
method BaseIndex.search_batch(queries, k: int=…) -> BatchedSearchResults → method BaseIndex.search_batch(queries, k: int=…, **kwargs) -> BatchedSearchResults - changed:
method ElasticSearchIndex.search(query: str, k=…) -> SearchResults → method ElasticSearchIndex.search(query: str, k=…, **kwargs) -> SearchResults - changed:
method ElasticSearchIndex.search_batch(queries, k: int=…, max_workers=…) -> BatchedSearchResults → method ElasticSearchIndex.search_batch(queries, k: int=…, max_workers=…, **kwargs) -> BatchedSearchResults - changed:
method FaissIndex.save(file: Union[str, PurePath]) → method FaissIndex.save(file: Union[str, PurePath], storage_options: Optional[Dict]=…) - changed:
method FaissIndex.search(query: np.array, k=…) -> SearchResults → method FaissIndex.search(query: np.array, k=…, **kwargs) -> SearchResults - changed:
method FaissIndex.search_batch(queries: np.array, k=…) -> BatchedSearchResults → method FaissIndex.search_batch(queries: np.array, k=…, **kwargs) -> BatchedSearchResults - changed:
method IndexableMixin.add_faiss_index(column: str, index_name: Optional[str]=…, device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, train_size: Optional[int]=…, faiss_verbose: bool=…) → method IndexableMixin.add_faiss_index(column: str, index_name: Optional[str]=…, device: Optional[Union[int, List[int]]]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, batch_size: int=…, train_size: Optional[int]=…, faiss_verbose: bool=…) - changed:
method IndexableMixin.add_faiss_index_from_external_arrays(external_arrays: np.array, index_name: str, device: Optional[int]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, train_size: Optional[int]=…, faiss_verbose: bool=…) → method IndexableMixin.add_faiss_index_from_external_arrays(external_arrays: np.array, index_name: str, device: Optional[Union[int, List[int]]]=…, string_factory: Optional[str]=…, metric_type: Optional[int]=…, custom_index: Optional[faiss.Index]=…, batch_size: int=…, train_size: Optional[int]=…, faiss_verbose: bool=…) - changed:
method IndexableMixin.get_nearest_examples(index_name: str, query: Union[str, np.array], k: int=…) -> NearestExamplesResults → method IndexableMixin.get_nearest_examples(index_name: str, query: Union[str, np.array], k: int=…, **kwargs) -> NearestExamplesResults - changed:
method IndexableMixin.get_nearest_examples_batch(index_name: str, queries: Union[List[str], np.array], k: int=…) -> BatchedNearestExamplesResults → method IndexableMixin.get_nearest_examples_batch(index_name: str, queries: Union[List[str], np.array], k: int=…, **kwargs) -> BatchedNearestExamplesResults - changed:
method IndexableMixin.load_faiss_index(index_name: str, file: Union[str, PurePath], device: Optional[int]=…) → method IndexableMixin.load_faiss_index(index_name: str, file: Union[str, PurePath], device: Optional[Union[int, List[int]]]=…, storage_options: Optional[Dict]=…) - changed:
method IndexableMixin.save_faiss_index(index_name: str, file: Union[str, PurePath]) → method IndexableMixin.save_faiss_index(index_name: str, file: Union[str, PurePath], storage_options: Optional[Dict]=…) - changed:
method IndexableMixin.search(index_name: str, query: Union[str, np.array], k: int=…) -> SearchResults → method IndexableMixin.search(index_name: str, query: Union[str, np.array], k: int=…, **kwargs) -> SearchResults - changed:
method IndexableMixin.search_batch(index_name: str, queries: Union[List[str], np.array], k: int=…) -> BatchedSearchResults → method IndexableMixin.search_batch(index_name: str, queries: Union[List[str], np.array], k: int=…, **kwargs) -> BatchedSearchResults - changed:
static method FaissIndex.load(file: Union[str, PurePath], device: Optional[int]=…) -> FaissIndex → static method FaissIndex.load(file: Union[str, PurePath], device: Optional[Union[int, List[int]]]=…, storage_options: Optional[Dict]=…) -> FaissIndex
- added:
field SplitInfo.shard_lengths: Optional[List[int]]
- changed:
function extend_module_for_streaming(module_path, use_auth_token: Optional[Union[str, bool]]=…) → function extend_module_for_streaming(module_path, download_config: Optional[DownloadConfig]=…) - added:
function extend_dataset_builder_for_streaming(builder: DatasetBuilder)
- removed:
method Table.select_columns(columns: List[int]) -> Table - removed:
variable IS_PYARROW_AT_LEAST_4 - changed:
function array_cast(array: pa.Array, pa_type: pa.DataType, allow_number_to_str=…) → function array_cast(array: pa.Array, pa_type: pa.DataType, allow_primitive_to_str: bool=…, allow_decimal_to_str: bool=…) -> Union[pa.Array, pa.FixedSizeListArray, pa.ListArray, pa.StructArray, pa.ExtensionArray] - changed:
function cast_array_to_feature(array: pa.Array, feature: FeatureType, allow_number_to_str=…) → function cast_array_to_feature(array: pa.Array, feature: FeatureType, allow_primitive_to_str: bool=…, allow_decimal_to_str: bool=…) -> pa.Array - added:
class CastError(ValueError) - added:
constructor CastError(*args, table_column_names: List[str], requested_column_names: List[str]) -> None - added:
function cast_table_to_schema(table: pa.Table, schema: pa.Schema) - added:
function embed_array_storage(array: pa.Array, feature: FeatureType) - added:
function embed_table_storage(table: pa.Table) - added:
function read_schema_from_file(filename: str) -> pa.Schema - added:
function table_flatten(table: pa.Table) - added:
function table_iter(table: Table, batch_size: int, drop_last_batch=…) -> Iterator[pa.Table] - added:
function table_visitor(table: pa.Table, function: Callable[[pa.Array], None]) - added:
method CastError.details() - added:
method ConcatenationTable.select(columns, *args, **kwargs) - added:
method InMemoryTable.select(*args, **kwargs) - added:
method MemoryMappedTable.select(*args, **kwargs) - added:
method Table.select(*args, **kwargs) - added:
method Table.to_pylist(*args, **kwargs) - added:
method Table.to_reader(max_chunksize: Optional[int]=…) - added:
static method InMemoryTable.from_pylist(mapping, *args, **kwargs)
- added:
export AudioClassification
- added:
class AudioClassification(TaskTemplate) - added:
field AudioClassification.audio_column: str - added:
field AudioClassification.input_schema: ClassVar[Features] - added:
field AudioClassification.label_column: str - added:
field AudioClassification.label_schema: ClassVar[Features] - added:
field AudioClassification.task: str - added:
method AudioClassification.align_with_features(features) - added:
property AudioClassification.column_mapping -> Dict[str, str]
- removed:
field AutomaticSpeechRecognition.audio_file_path_column: str - added:
field AutomaticSpeechRecognition.audio_column: str - added:
method AutomaticSpeechRecognition.align_with_features(features)
- removed:
export DownloadConfig - removed:
export DownloadManager - removed:
export GenerateMode - removed:
export MockDownloadManager - removed:
export NonMutableDict - removed:
export cached_path - removed:
export classproperty - removed:
export copyfunc - removed:
export dumps - removed:
export first_non_null_value - removed:
export flatten_nest_dict - removed:
export has_sufficient_disk_space - removed:
export hf_bucket_url - removed:
export is_remote_url - removed:
export logging - removed:
export map_nested - removed:
export memoize - removed:
export no_op_if_value_is_null - removed:
export relative_to_absolute_path - removed:
export set_progress_bar_enabled - removed:
export size_str - removed:
export temp_seed - removed:
export temporary_assignment - removed:
export unique_values - removed:
export zip_dict - removed:
export zip_nested - added:
export VerificationMode - added:
export are_progress_bars_disabled - added:
export disable_progress_bars - added:
export enable_progress_bar - added:
export enable_progress_bars - added:
export experimental
- removed:
class WriteToParquet(PTransform) - removed:
constructor WriteToParquet(file_path_prefix, schema, row_group_buffer_size=…, record_batch_size=…, codec=…, use_deprecated_int96_timestamps=…, file_name_suffix=…, num_shards=…, shard_name_template=…, mime_type=…) - removed:
method WriteToParquet.display_data() - removed:
method WriteToParquet.expand(pcoll)
- added:
function cached_path(url_or_filename, download_config=…, **download_kwargs) -> str - added:
function get_from_cache(url, cache_dir=…, force_download=…, proxies=…, etag_timeout=…, resume_download=…, user_agent=…, local_files_only=…, use_etag=…, max_retries=…, use_auth_token=…, ignore_url_params=…, download_desc=…) -> str
- added:
class DeprecatedEnum(enum.Enum) - added:
class OnAccess(enum.EnumMeta) - added:
method DeprecatedEnum.deprecate() - added:
property DeprecatedEnum.help_message
- removed:
class DownloadManager - removed:
class GenerateMode(enum.Enum) - removed:
constructor DownloadManager(dataset_name: Optional[str]=…, data_dir: Optional[str]=…, download_config: Optional[DownloadConfig]=…, base_path: Optional[str]=…) - removed:
field GenerateMode.FORCE_REDOWNLOAD - removed:
field GenerateMode.REUSE_CACHE_IF_EXISTS - removed:
field GenerateMode.REUSE_DATASET_IF_EXISTS - removed:
method DownloadManager.delete_extracted_files() - removed:
method DownloadManager.download(url_or_urls) - removed:
method DownloadManager.download_and_extract(url_or_urls) - removed:
method DownloadManager.download_custom(url_or_urls, custom_download) - removed:
method DownloadManager.extract(path_or_paths, num_proc=…) - removed:
method DownloadManager.get_recorded_sizes_checksums() - removed:
method DownloadManager.iter_archive(path_or_buf: Union[str, io.BufferedReader]) - removed:
method DownloadManager.iter_files(paths) - removed:
method DownloadManager.manage_extracted_files() - removed:
method DownloadManager.ship_files_with_pipeline(downloaded_path_or_paths, pipeline) - removed:
property DownloadManager.downloaded_size - removed:
property DownloadManager.manual_dir - removed:
variable logger
- added:
function experimental(fn: Callable) -> Callable
- removed:
static method GzipExtractor.is_extractable(path: str) -> bool - removed:
static method RarExtractor.is_extractable(path: str) -> bool - removed:
static method XzExtractor.is_extractable(path: str) -> bool - removed:
static method ZstdExtractor.is_extractable(path: str) -> bool - changed:
class GzipExtractor → class GzipExtractor(MagicNumberBaseExtractor) - changed:
class RarExtractor → class RarExtractor(MagicNumberBaseExtractor) - changed:
class TarExtractor → class TarExtractor(BaseExtractor) - changed:
class XzExtractor → class XzExtractor(MagicNumberBaseExtractor) - changed:
class ZipExtractor → class ZipExtractor(MagicNumberBaseExtractor) - changed:
class ZstdExtractor → class ZstdExtractor(MagicNumberBaseExtractor) - changed:
constructor ExtractManager(cache_dir=…) → constructor ExtractManager(cache_dir: Optional[str]=…) - changed:
field Extractor.extractors → field Extractor.extractors: Dict[str, Type[BaseExtractor]] - changed:
method ExtractManager.extract(input_path, force_extract=…) → method ExtractManager.extract(input_path: str, force_extract: bool=…) -> str - changed:
static method Extractor.extract(input_path, output_path, extractor=…) → static method Extractor.extract(input_path: Union[Path, str], output_path: Union[Path, str], extractor_format: Optional[str]=…, extractor: Optional[BaseExtractor]=…) -> None - changed:
static method Extractor.is_extractable(path, return_extractor=…) → static method Extractor.is_extractable(path: Union[Path, str], return_extractor: bool=…) -> bool - changed:
static method GzipExtractor.extract(input_path, output_path) → static method GzipExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - changed:
static method RarExtractor.extract(input_path, output_path) → static method RarExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - changed:
static method TarExtractor.extract(input_path, output_path) → static method TarExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - changed:
static method TarExtractor.is_extractable(path) → static method TarExtractor.is_extractable(path: Union[Path, str], **kwargs) -> bool - changed:
static method XzExtractor.extract(input_path, output_path) → static method XzExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - changed:
static method ZipExtractor.extract(input_path, output_path) → static method ZipExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - changed:
static method ZipExtractor.is_extractable(path) → static method ZipExtractor.is_extractable(path: Union[Path, str], magic_number: bytes=…) -> bool - changed:
static method ZstdExtractor.extract(input_path: str, output_path: str) → static method ZstdExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - added:
class BaseExtractor(ABC) - added:
class Bzip2Extractor(MagicNumberBaseExtractor) - added:
class Lz4Extractor(MagicNumberBaseExtractor) - added:
class MagicNumberBaseExtractor(BaseExtractor, ABC) - added:
class SevenZipExtractor(MagicNumberBaseExtractor) - added:
field Bzip2Extractor.magic_numbers - added:
field GzipExtractor.magic_numbers - added:
field Lz4Extractor.magic_numbers - added:
field MagicNumberBaseExtractor.magic_numbers: List[bytes] - added:
field RarExtractor.magic_numbers - added:
field SevenZipExtractor.magic_numbers - added:
field XzExtractor.magic_numbers - added:
field ZipExtractor.magic_numbers - added:
field ZstdExtractor.magic_numbers - added:
static method BaseExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - added:
static method BaseExtractor.is_extractable(path: Union[Path, str], **kwargs) -> bool - added:
static method Bzip2Extractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - added:
static method Extractor.infer_extractor_format(path: Union[Path, str]) -> Optional[str] - added:
static method Lz4Extractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - added:
static method MagicNumberBaseExtractor.is_extractable(path: Union[Path, str], magic_number: bytes=…) -> bool - added:
static method MagicNumberBaseExtractor.read_magic_number(path: Union[Path, str], magic_number_length: int) - added:
static method SevenZipExtractor.extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None - added:
static method TarExtractor.safemembers(members, output_path) - added:
variable logger
- removed:
class DownloadConfig - removed:
field DownloadConfig.cache_dir: Optional[Union[str, Path]] - removed:
field DownloadConfig.delete_extracted: bool - removed:
field DownloadConfig.extract_compressed_file: bool - removed:
field DownloadConfig.force_download: bool - removed:
field DownloadConfig.force_extract: bool - removed:
field DownloadConfig.ignore_url_params: bool - removed:
field DownloadConfig.local_files_only: bool - removed:
field DownloadConfig.max_retries: int - removed:
field DownloadConfig.num_proc: Optional[int] - removed:
field DownloadConfig.proxies: Optional[Dict] - removed:
field DownloadConfig.resume_download: bool - removed:
field DownloadConfig.use_auth_token: Optional[Union[str, bool]] - removed:
field DownloadConfig.use_etag: bool - removed:
field DownloadConfig.user_agent: Optional[str] - removed:
function hf_hub_url(path: str, name: str, revision: Optional[str]=…, version=…) -> str - removed:
function temp_seed(seed: int, set_pytorch=…, set_tensorflow=…) - removed:
method DownloadConfig.copy() -> DownloadConfig - changed:
function get_authentication_headers_for_url(url: str, use_auth_token: Optional[Union[str, bool]]=…) -> dict → function get_authentication_headers_for_url(url: str, token: Optional[Union[str, bool]]=…, use_auth_token: Optional[Union[str, bool]]=…) -> dict - changed:
function get_from_cache(url, cache_dir=…, force_download=…, proxies=…, etag_timeout=…, resume_download=…, user_agent=…, local_files_only=…, use_etag=…, max_retries=…, use_auth_token=…, ignore_url_params=…) -> str → function get_from_cache(url, cache_dir=…, force_download=…, proxies=…, etag_timeout=…, resume_download=…, user_agent=…, local_files_only=…, use_etag=…, max_retries=…, token=…, use_auth_token=…, ignore_url_params=…, storage_options=…, download_desc=…, disable_tqdm=…) -> str - changed:
function hf_github_url(path: str, name: str, dataset=…, revision: Optional[str]=…, version=…) -> str → function hf_github_url(path: str, name: str, dataset=…, revision: Optional[str]=…) -> str - changed:
function http_get(url, temp_file, proxies=…, resume_size=…, headers=…, cookies=…, timeout=…, max_retries=…) → function http_get(url, temp_file, proxies=…, resume_size=…, headers=…, cookies=…, timeout=…, max_retries=…, desc=…, disable_tqdm=…) -> Optional[requests.Response] - changed:
function request_etag(url: str, use_auth_token: Optional[Union[str, bool]]=…) -> Optional[str] → function request_etag(url: str, token: Optional[Union[str, bool]]=…, use_auth_token: Optional[Union[str, bool]]=…) -> Optional[str] - added:
class ArchiveIterable(TrackedIterableFromGenerator) - added:
class FilesIterable(TrackedIterableFromGenerator) - added:
class NonStreamableDatasetError(Exception) - added:
class TqdmCallback(fsspec.callbacks.TqdmCallback) - added:
class xPath(type(Path())) - added:
constructor TqdmCallback(tqdm_kwargs=…, *args, **kwargs) - added:
function fsspec_get(url, temp_file, storage_options=…, desc=…, disable_tqdm=…) - added:
function fsspec_head(url, storage_options=…) - added:
function stack_multiprocessing_download_progress_bars() - added:
function xbasename(a) - added:
function xdirname(a) - added:
function xet_parse(source, parser=…, download_config: Optional[DownloadConfig]=…) - added:
function xexists(urlpath: str, download_config: Optional[DownloadConfig]=…) - added:
function xgetsize(path, download_config: Optional[DownloadConfig]=…) -> int - added:
function xglob(urlpath, *, recursive=…, download_config: Optional[DownloadConfig]=…) - added:
function xgzip_open(filepath_or_buffer, *args, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xisdir(path, download_config: Optional[DownloadConfig]=…) -> bool - added:
function xisfile(path, download_config: Optional[DownloadConfig]=…) -> bool - added:
function xjoin(a, *p) - added:
function xlistdir(path: str, download_config: Optional[DownloadConfig]=…) -> List[str] - added:
function xnumpy_load(filepath_or_buffer, *args, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xopen(file: str, mode=…, *args, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xpandas_read_csv(filepath_or_buffer, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xpandas_read_excel(filepath_or_buffer, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xpyarrow_parquet_read_table(filepath_or_buffer, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xrelpath(path, start=…) - added:
function xsio_loadmat(filepath_or_buffer, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xsplit(a) - added:
function xsplitext(a) - added:
function xwalk(urlpath, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
function xxml_dom_minidom_parse(filename_or_file, download_config: Optional[DownloadConfig]=…, **kwargs) - added:
method xPath.exists(download_config: Optional[DownloadConfig]=…) - added:
method xPath.glob(pattern, download_config: Optional[DownloadConfig]=…) - added:
method xPath.joinpath(*p: Tuple[str, ...]) -> xPath - added:
method xPath.open(*args, **kwargs) - added:
method xPath.rglob(pattern, **kwargs) - added:
method xPath.with_suffix(suffix) - added:
property xPath.name -> str - added:
property xPath.parent -> xPath - added:
property xPath.stem -> str - added:
property xPath.suffix -> str - added:
static method ArchiveIterable.from_buf(fileobj) -> ArchiveIterable - added:
static method ArchiveIterable.from_urlpath(urlpath_or_buf, download_config: Optional[DownloadConfig]=…) -> ArchiveIterable - added:
static method FilesIterable.from_urlpaths(urlpaths, download_config: Optional[DownloadConfig]=…) -> FilesIterable - added:
variable BASE_KNOWN_EXTENSIONS - added:
variable COMPRESSION_EXTENSION_TO_PROTOCOL - added:
variable MAGIC_NUMBER_MAX_LENGTH - added:
variable MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL - added:
variable MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL - added:
variable SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL - added:
variable SINGLE_FILE_COMPRESSION_PROTOCOLS - added:
variable SINGLE_SLASH_AFTER_PROTOCOL_PATTERN
- removed:
class BaseFileLock - removed:
class SoftFileLock(BaseFileLock) - removed:
class Timeout(TimeoutError) - removed:
class UnixFileLock(BaseFileLock) - removed:
class WindowsFileLock(BaseFileLock) - removed:
constructor BaseFileLock(lock_file, timeout=…, max_filename_length=…) - removed:
constructor Timeout(lock_file) - removed:
constructor UnixFileLock(lock_file, timeout=…, max_filename_length=…) - removed:
constructor WindowsFileLock(lock_file, timeout=…, max_filename_length=…) - removed:
method BaseFileLock.acquire(timeout=…, poll_intervall=…) - removed:
method BaseFileLock.hash_filename_if_too_long(path: str, max_length: int) -> str - removed:
method BaseFileLock.release(force=…) - removed:
property BaseFileLock.is_locked - removed:
property BaseFileLock.lock_file - removed:
property BaseFileLock.timeout - removed:
variable FileLock
- added:
variable hf_dataset_url
- removed:
class ChecksumVerificationException(Exception) - removed:
class ExpectedMoreDownloadedFiles(ChecksumVerificationException) - removed:
class ExpectedMoreSplits(SplitsVerificationException) - removed:
class NonMatchingChecksumError(ChecksumVerificationException) - removed:
class NonMatchingSplitsSizesError(SplitsVerificationException) - removed:
class SplitsVerificationException(Exception) - removed:
class UnexpectedDownloadedFile(ChecksumVerificationException) - removed:
class UnexpectedSplits(SplitsVerificationException) - changed:
function get_size_checksum_dict(path: str) -> dict → function get_size_checksum_dict(path: str, record_checksum: bool=…) -> dict - added:
class VerificationMode(enum.Enum) - added:
field VerificationMode.ALL_CHECKS - added:
field VerificationMode.BASIC_CHECKS - added:
field VerificationMode.NO_CHECKS
- removed:
class NoDuplicateSafeLoader(yaml.SafeLoader) - removed:
field DatasetMetadata.annotations_creators: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.language_creators: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.languages: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.licenses: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.multilinguality: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.paperswithcode_id: Optional[str] - removed:
field DatasetMetadata.pretty_name: Union[str, Dict[str, str]] - removed:
field DatasetMetadata.size_categories: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.source_datasets: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.task_categories: Union[List[str], Dict[str, List[str]]] - removed:
field DatasetMetadata.task_ids: Union[List[str], Dict[str, List[str]]] - removed:
function load_json_resource(resource: str) -> Tuple[Any, str] - removed:
function metadata_dict_from_readme(path: Path) -> Optional[Dict[str, List[str]]] - removed:
function tagset_validator(items: Union[List[str], Dict[str, List[str]]], reference_values: List[str], name: str, url: str, escape_validation_predicate_fn: Optional[Callable[[Any], bool]]=…) -> ValidatorOutput - removed:
function validate_metadata_type(metadata_dict: dict) - removed:
function validate_type(value: Any, expected_type: Type) - removed:
function yaml_block_from_readme(path: Path) -> Optional[str] - removed:
method DatasetMetadata.get_metadata_by_config_name(name: str) -> DatasetMetadata - removed:
method DatasetMetadata.validate() - removed:
method NoDuplicateSafeLoader.construct_mapping(node, deep=…) - removed:
static method DatasetMetadata.validate_annotations_creators(annotations_creators: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_language_codes(languages: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_language_creators(language_creators: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_licences(licenses: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_mulitlinguality(multilinguality: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_paperswithcode_id_errors(paperswithcode_id: Optional[str]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_pretty_name(pretty_name: Union[str, Dict[str, str]]) - removed:
static method DatasetMetadata.validate_size_catgeories(size_cats: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_source_datasets(sources: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_task_categories(task_categories: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
static method DatasetMetadata.validate_task_ids(task_ids: Union[List[str], Dict[str, List[str]]]) -> ValidatorOutput - removed:
variable BASE_REF_URL - removed:
variable ValidatorOutput - removed:
variable known_creators - removed:
variable known_creators_url - removed:
variable known_language_codes - removed:
variable known_language_codes_url - removed:
variable known_licenses - removed:
variable known_licenses_url - removed:
variable known_multilingualities - removed:
variable known_multilingualities_url - removed:
variable known_size_categories - removed:
variable known_size_categories_url - removed:
variable known_source_datasets - removed:
variable known_source_datasets_url - removed:
variable known_task_ids_url - removed:
variable this_url - changed:
class DatasetMetadata → class DatasetMetadata(dict) - changed:
static method DatasetMetadata.from_readme(path: Path) -> DatasetMetadata → static method DatasetMetadata.from_readme(path: Union[Path, str]) -> DatasetMetadata - added:
class MetadataConfigs(Dict[str, Dict[str, Any]]) - added:
field MetadataConfigs.FIELD_NAME: ClassVar[str] - added:
method DatasetMetadata.to_readme(path: Path) - added:
method DatasetMetadata.to_yaml_string() -> str - added:
method MetadataConfigs.get_default_config_name() -> Optional[str] - added:
method MetadataConfigs.to_dataset_card_data(dataset_card_data: DatasetCardData) -> None - added:
static method MetadataConfigs.from_dataset_card_data(dataset_card_data: DatasetCardData) -> MetadataConfigs
- removed:
class MockDownloadManager - removed:
constructor MockDownloadManager(dataset_name: str, config: str, version: Union[Version, str], cache_dir: Optional[str]=…, use_local_dummy_data: bool=…, load_existing_dummy_data: bool=…, download_callbacks: Optional[List[Callable]]=…) - removed:
field MockDownloadManager.datasets_scripts_dir - removed:
field MockDownloadManager.dummy_file_name - removed:
method MockDownloadManager.create_dummy_data_dict(path_to_dummy_data, data_url) - removed:
method MockDownloadManager.create_dummy_data_list(path_to_dummy_data, data_url) - removed:
method MockDownloadManager.create_dummy_data_single(path_to_dummy_data, data_url) - removed:
method MockDownloadManager.delete_extracted_files() - removed:
method MockDownloadManager.download(data_url, *args) - removed:
method MockDownloadManager.download_and_extract(data_url, *args) - removed:
method MockDownloadManager.download_custom(data_url, custom_download) - removed:
method MockDownloadManager.download_dummy_data() - removed:
method MockDownloadManager.extract(path, *args, **kwargs) - removed:
method MockDownloadManager.get_recorded_sizes_checksums() - removed:
method MockDownloadManager.iter_archive(path) - removed:
method MockDownloadManager.iter_files(paths) - removed:
method MockDownloadManager.manage_extracted_files() - removed:
property MockDownloadManager.dummy_data_folder - removed:
property MockDownloadManager.dummy_file - removed:
property MockDownloadManager.dummy_zip_file - removed:
property MockDownloadManager.github_path_to_dummy_data - removed:
property MockDownloadManager.local_path_to_dummy_data - removed:
property MockDownloadManager.manual_dir - removed:
variable logger
- removed:
class Pickler(dill.Pickler) - removed:
field Pickler.dispatch - removed:
function dump(obj, file) - removed:
function dumps(obj) - removed:
function flatten_nest_dict(d) - removed:
function pklregister(t) - removed:
function save_function(pickler, obj) - removed:
function zip_nested(arg0, *args, **kwargs) - removed:
method Pickler.memoize(obj) - removed:
method Pickler.save_global(obj, name=…) - changed:
function map_nested(function, data_struct, dict_only: bool=…, map_list: bool=…, map_tuple: bool=…, map_numpy: bool=…, num_proc: Optional[int]=…, types=…, disable_tqdm: bool=…) → function map_nested(function: Callable[[Any], Any], data_struct: Any, dict_only: bool=…, map_list: bool=…, map_tuple: bool=…, map_numpy: bool=…, num_proc: Optional[int]=…, parallel_min_length: int=…, batched: bool=…, batch_size: Optional[int]=…, types: Optional[tuple]=…, disable_tqdm: bool=…, desc: Optional[str]=…) -> Any - added:
function asdict(obj) - added:
function convert_file_size_to_int(size: Union[int, str]) -> int - added:
function get_imports(file_path: str) -> Tuple[str, str, str, str] - added:
function glob_pattern_to_regex(pattern) - added:
function iflatmap_unordered(pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool], func: Callable[..., Iterable[Y]], *, kwargs_iterable: Iterable[dict]) -> Iterable[Y] - added:
function iter_batched(iterable: Iterable[T], n: int) -> Iterable[List[T]] - added:
function lock_importable_file(importable_local_file: str) -> FileLock - added:
function temp_seed(seed: int, set_pytorch=…, set_tensorflow=…) - added:
variable T - added:
variable Y
- added:
function approximate_mode(class_counts, n_draws, rng) - added:
function stratified_shuffle_split_generate_indices(y, n_train, n_test, rng, n_splits=…)
- removed:
class ArchiveIterable(_IterableFromGenerator) - removed:
class FilesIterable(_IterableFromGenerator) - removed:
class NonStreamableDatasetError(Exception) - removed:
class StreamingDownloadManager - removed:
constructor StreamingDownloadManager(dataset_name: Optional[str]=…, data_dir: Optional[str]=…, download_config: Optional[DownloadConfig]=…, base_path: Optional[str]=…) - removed:
function xbasename(a) - removed:
function xdirname(a) - removed:
function xet_parse(source, parser=…, use_auth_token: Optional[Union[str, bool]]=…) - removed:
function xglob(urlpath, *, recursive=…, use_auth_token: Optional[Union[str, bool]]=…) - removed:
function xisdir(path, use_auth_token: Optional[Union[str, bool]]=…) -> bool - removed:
function xisfile(path, use_auth_token: Optional[Union[str, bool]]=…) -> bool - removed:
function xjoin(a, *p) - removed:
function xlistdir(path: str, use_auth_token: Optional[Union[str, bool]]=…) -> List[str] - removed:
function xopen(file: str, mode=…, *args, use_auth_token: Optional[Union[str, bool]]=…, **kwargs) - removed:
function xpandas_read_csv(filepath_or_buffer, use_auth_token: Optional[Union[str, bool]]=…, **kwargs) - removed:
function xpandas_read_excel(filepath_or_buffer, **kwargs) - removed:
function xpathglob(path, pattern, use_auth_token: Optional[Union[str, bool]]=…) - removed:
function xpathjoin(a: Path, *p: Tuple[str, ...]) - removed:
function xpathname(path: Path) - removed:
function xpathopen(path: Path, *args, **kwargs) - removed:
function xpathparent(path: Path) - removed:
function xpathrglob(path, pattern, **kwargs) - removed:
function xpathstem(path: Path) - removed:
function xpathsuffix(path: Path) - removed:
function xsio_loadmat(filepath_or_buffer, use_auth_token: Optional[Union[str, bool]]=…, **kwargs) - removed:
function xsplitext(a) - removed:
function xwalk(urlpath, use_auth_token: Optional[Union[str, bool]]=…) - removed:
method StreamingDownloadManager.download(url_or_urls) - removed:
method StreamingDownloadManager.download_and_extract(url_or_urls) - removed:
method StreamingDownloadManager.extract(path_or_paths) - removed:
method StreamingDownloadManager.iter_archive(urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[Tuple] - removed:
method StreamingDownloadManager.iter_files(urlpaths: List[str]) -> Iterable[str] - removed:
property StreamingDownloadManager.manual_dir - removed:
static method ArchiveIterable.from_buf(fileobj) -> ArchiveIterable - removed:
static method ArchiveIterable.from_urlpath(urlpath_or_buf, use_auth_token: Optional[Union[str, bool]]=…) -> ArchiveIterable - removed:
static method FilesIterable.from_urlpaths(urlpaths, use_auth_token: Optional[Union[str, bool]]=…) -> FilesIterable - removed:
variable BASE_KNOWN_EXTENSIONS - removed:
variable COMPRESSION_EXTENSION_TO_PROTOCOL - removed:
variable MAGIC_NUMBER_MAX_LENGTH - removed:
variable MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL - removed:
variable MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL - removed:
variable SINGLE_FILE_COMPRESSION_PROTOCOLS - removed:
variable SINGLE_SLASH_AFTER_PROTOCOL_PATTERN - removed:
variable logger
- added:
class NumpyMultiprocessingGenerator - added:
class SharedMemoryContext - added:
constructor NumpyMultiprocessingGenerator(dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, output_signature, shuffle, batch_size, drop_remainder, num_workers) - added:
constructor SharedMemoryContext() - added:
function dataset_to_tf(dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, output_signature, shuffle, batch_size, drop_remainder) - added:
function is_numeric_pa_type(pa_type) - added:
function minimal_tf_collate_fn(features) - added:
function minimal_tf_collate_fn_with_renaming(features) - added:
function multiprocess_dataset_to_tf(dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, output_signature, shuffle, batch_size, drop_remainder, num_workers) - added:
function np_get_batch(indices, dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, return_dict=…) - added:
method SharedMemoryContext.get_array(name, shape, dtype, create) - added:
method SharedMemoryContext.get_shm(name, size, create) - added:
static method NumpyMultiprocessingGenerator.distribute_batches(dataset, batch_size, drop_remainder, num_workers, shuffle) - added:
static method NumpyMultiprocessingGenerator.worker_loop(dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, columns_to_ranks, string_columns, indices, extra_batch, worker_name, array_ready_event, array_loaded_event) - added:
variable SharedMemory
- added:
class tqdm(old_tqdm) - added:
constructor tqdm(*args, **kwargs) - added:
function are_progress_bars_disabled() -> bool - added:
function disable_progress_bars() -> None - added:
function enable_progress_bars() -> None - added:
function is_progress_bar_enabled() - added:
variable disable_progress_bar - added:
variable enable_progress_bar
- removed:
class EmptyTqdm - removed:
constructor EmptyTqdm(*args, **kwargs) - removed:
function disable_progress_bar() - removed:
function is_progress_bar_enabled() -> bool - removed:
function set_progress_bar_enabled(boolean: bool) - removed:
variable tqdm
- added:
class TrackedIterableFromGenerator(Iterable) - added:
class tracked_list(list) - added:
class tracked_str(str) - added:
constructor TrackedIterableFromGenerator(generator, *args) - added:
constructor tracked_list(*args, **kwargs) -> None - added:
field tracked_str.origins - added:
method tracked_str.get_origin() - added:
method tracked_str.set_origin(origin: str)
- added:
variable ListLike
- removed:
method Version.match(other_version)