swh.storage.in_memory module

class swh.storage.in_memory.Table(*args, **kwds)

Bases: typing.Generic

partition_key(row: Union[TRow, Dict[str, Any]]) → Tuple

Returns the partition key of a row (ie. the cells which get hashed into the token.

clustering_key(row: Union[TRow, Dict[str, Any]]) → Tuple

Returns the clustering key of a row (ie. the cells which are used for sorting rows within a partition.

primary_key(row)

primary_key_from_dict(d: Dict[str, Any]) → Tuple

Returns the primary key (ie. concatenation of partition key and clustering key) of the given dictionary interpreted as a row.

token(key: Tuple)

Returns the token of a row (ie. the hash of its partition key).

get_partition(token: int) → Dict[Tuple, TRow]

Returns the partition that contains this token.

insert(row: TRow)

split_primary_key(key: Tuple) → Tuple[Tuple, Tuple]

Returns (partition_key, clustering_key) from a partition key

get_from_partition_key(partition_key: Tuple) → Iterable[TRow]

Returns at most one row, from its partition key.

get_from_primary_key(primary_key: Tuple) → Optional[TRow]

Returns at most one row, from its primary key.

get_from_token(token: int) → Iterable[TRow]

Returns all rows whose token (ie. non-cryptographic hash of the partition key) is the one passed as argument.

iter_all() → Iterator[Tuple[Tuple, TRow]]

get_random() → Optional[TRow]

class swh.storage.in_memory.InMemoryCqlRunner

Bases: object

increment_counter(object_type: str, nb: int)

stat_counters() → Iterable[swh.storage.cassandra.model.ObjectCountRow]

content_add_prepare(content: swh.storage.cassandra.model.ContentRow)

content_get_from_pk(content_hashes: Dict[str, bytes]) → Optional[swh.storage.cassandra.model.ContentRow]

content_get_from_token(token: int) → Iterable[swh.storage.cassandra.model.ContentRow]

content_get_random() → Optional[swh.storage.cassandra.model.ContentRow]

content_get_token_range(start: int, end: int, limit: int) → Iterable[Tuple[int, swh.storage.cassandra.model.ContentRow]]

content_missing_by_sha1_git(ids: List[bytes]) → List[bytes]

content_index_add_one(algo: str, content: swh.model.model.Content, token: int) → None

content_get_tokens_from_single_hash(algo: str, hash_: bytes) → Iterable[int]

skipped_content_add_prepare(content: swh.storage.cassandra.model.SkippedContentRow)

skipped_content_get_from_pk(content_hashes: Dict[str, bytes]) → Optional[swh.storage.cassandra.model.SkippedContentRow]

skipped_content_get_from_token(token: int) → Iterable[swh.storage.cassandra.model.SkippedContentRow]

skipped_content_index_add_one(algo: str, content: swh.model.model.SkippedContent, token: int) → None

skipped_content_get_tokens_from_single_hash(algo: str, hash_: bytes) → Iterable[int]

directory_missing(ids: List[bytes]) → List[bytes]

directory_add_one(directory: swh.storage.cassandra.model.DirectoryRow) → None

directory_get_random() → Optional[swh.storage.cassandra.model.DirectoryRow]

directory_entry_add_one(entry: swh.storage.cassandra.model.DirectoryEntryRow) → None

directory_entry_get(directory_ids: List[bytes]) → Iterable[swh.storage.cassandra.model.DirectoryEntryRow]

revision_missing(ids: List[bytes]) → Iterable[bytes]

revision_add_one(revision: swh.storage.cassandra.model.RevisionRow) → None

revision_get_ids(revision_ids) → Iterable[int]

revision_get(revision_ids: List[bytes]) → Iterable[swh.storage.cassandra.model.RevisionRow]

revision_get_random() → Optional[swh.storage.cassandra.model.RevisionRow]

revision_parent_add_one(revision_parent: swh.storage.cassandra.model.RevisionParentRow) → None

revision_parent_get(revision_id: bytes) → Iterable[bytes]

release_missing(ids: List[bytes]) → List[bytes]

release_add_one(release: swh.storage.cassandra.model.ReleaseRow) → None

release_get(release_ids: List[str]) → Iterable[swh.storage.cassandra.model.ReleaseRow]

release_get_random() → Optional[swh.storage.cassandra.model.ReleaseRow]

snapshot_missing(ids: List[bytes]) → List[bytes]

snapshot_add_one(snapshot: swh.storage.cassandra.model.SnapshotRow) → None

snapshot_get_random() → Optional[swh.storage.cassandra.model.SnapshotRow]

snapshot_branch_add_one(branch: swh.storage.cassandra.model.SnapshotBranchRow) → None

snapshot_count_branches(snapshot_id: bytes) → Dict[Optional[str], int]

Returns a dictionary from type names to the number of branches of that type.

snapshot_branch_get(snapshot_id: bytes, from_: bytes, limit: int) → Iterable[swh.storage.cassandra.model.SnapshotBranchRow]

origin_add_one(origin: swh.storage.cassandra.model.OriginRow) → None

origin_get_by_sha1(sha1: bytes) → Iterable[swh.storage.cassandra.model.OriginRow]

origin_get_by_url(url: str) → Iterable[swh.storage.cassandra.model.OriginRow]

origin_list(start_token: int, limit: int) → Iterable[Tuple[int, swh.storage.cassandra.model.OriginRow]]

Returns an iterable of (token, origin)

origin_iter_all() → Iterable[swh.storage.cassandra.model.OriginRow]

origin_generate_unique_visit_id(origin_url: str) → int

origin_visit_get(origin_url: str, last_visit: Optional[int], limit: int, order: swh.storage.interface.ListOrder) → Iterable[swh.storage.cassandra.model.OriginVisitRow]

origin_visit_add_one(visit: swh.storage.cassandra.model.OriginVisitRow) → None

origin_visit_get_one(origin_url: str, visit_id: int) → Optional[swh.storage.cassandra.model.OriginVisitRow]

origin_visit_get_all(origin_url: str) → Iterable[swh.storage.cassandra.model.OriginVisitRow]

origin_visit_iter(start_token: int) → Iterator[swh.storage.cassandra.model.OriginVisitRow]

Returns all origin visits in order from this token, and wraps around the token space.

origin_visit_status_get_range(origin: str, visit: int, date_from: Optional[datetime.datetime], limit: int, order: swh.storage.interface.ListOrder) → Iterable[swh.storage.cassandra.model.OriginVisitStatusRow]

origin_visit_status_add_one(visit_update: swh.storage.cassandra.model.OriginVisitStatusRow) → None

origin_visit_status_get_latest(origin: str, visit: int) → Optional[swh.storage.cassandra.model.OriginVisitStatusRow]

Given an origin visit id, return its latest origin_visit_status

origin_visit_status_get(origin: str, visit: int) → Iterator[swh.storage.cassandra.model.OriginVisitStatusRow]

Return all origin visit statuses for a given visit

metadata_authority_add(authority: swh.storage.cassandra.model.MetadataAuthorityRow)

metadata_authority_get(type, url) → Optional[swh.storage.cassandra.model.MetadataAuthorityRow]

metadata_fetcher_add(fetcher: swh.storage.cassandra.model.MetadataFetcherRow)

metadata_fetcher_get(name, version) → Optional[swh.storage.cassandra.model.MetadataAuthorityRow]

raw_extrinsic_metadata_add(raw_extrinsic_metadata)

raw_extrinsic_metadata_get_after_date(id: str, authority_type: str, authority_url: str, after: datetime.datetime) → Iterable[swh.storage.cassandra.model.RawExtrinsicMetadataRow]

raw_extrinsic_metadata_get_after_date_and_fetcher(id: str, authority_type: str, authority_url: str, after_date: datetime.datetime, after_fetcher_name: str, after_fetcher_version: str) → Iterable[swh.storage.cassandra.model.RawExtrinsicMetadataRow]

raw_extrinsic_metadata_get(id: str, authority_type: str, authority_url: str) → Iterable[swh.storage.cassandra.model.RawExtrinsicMetadataRow]

class swh.storage.in_memory.InMemoryStorage(journal_writer=None)

Bases: swh.storage.cassandra.storage.CassandraStorage

reset()

check_config(*, check_write: bool) → bool