swh.storage.postgresql.db module

swh.storage.postgresql.db.jsonize(d)

class swh.storage.postgresql.db.Db(conn: connection, pool: Optional[AbstractConnectionPool] = None)

Bases: BaseDb

Proxy to the SWH DB, with wrappers around stored procedures

create a DB proxy

Parameters:

• conn – psycopg2 connection to the SWH DB

• pool – psycopg2 pool of connections

mktemp_dir_entry(entry_type, cur=None)

mktemp_revision(cur=None)

mktemp_release(cur=None)

mktemp_snapshot_branch(cur=None)

content_add_from_temp(cur=None)

directory_add_from_temp(cur=None)

skipped_content_add_from_temp(cur=None)

revision_add_from_temp(cur=None)

extid_add_from_temp(cur=None)

release_add_from_temp(cur=None)

content_update_from_temp(keys_to_update, cur=None)

content_get_metadata_keys = ['sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'status']

content_add_keys = ['sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'status', 'ctime']

skipped_content_keys = ['sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'reason', 'status', 'origin']

content_get_metadata_from_hashes(hashes: List[bytes], algo: str, cur=None)

content_get_range(start, end, limit=None, cur=None) → Iterator[Tuple]

Retrieve contents within range [start, end].

content_hash_keys = ['sha1', 'sha1_git', 'sha256', 'blake2s256']

content_missing_from_list(contents, cur=None)

content_missing_per_sha1(sha1s, cur=None)

content_missing_per_sha1_git(contents, cur=None)

content_find_cols = ['sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'ctime', 'status']

content_find(sha1: Optional[bytes] = None, sha1_git: Optional[bytes] = None, sha256: Optional[bytes] = None, blake2s256: Optional[bytes] = None, cur=None)

Find the content optionally on a combination of the following checksums sha1, sha1_git, sha256 or blake2s256.

Parameters:

• sha1 – sha1 content

• git_sha1 – the sha1 computed a la git sha1 of the content

• sha256 – sha256 content

• blake2s256 – blake2s256 content

Returns:

The tuple (sha1, sha1_git, sha256, blake2s256) if found or None.

content_get_random(cur=None)

skipped_content_missing(contents, cur=None)

directory_missing_from_list(directories, cur=None)

directory_ls_cols = ['dir_id', 'type', 'target', 'name', 'perms', 'status', 'sha1', 'sha1_git', 'sha256', 'length']

directory_walk_one(directory, cur=None)

directory_walk(directory, cur=None)

directory_entry_get_by_path(directory, paths, cur=None)

Retrieve a directory entry by path.

directory_get_entries_cols = ['type', 'target', 'name', 'perms']

directory_get_entries(directory: bytes, cur=None) → List[Tuple]

directory_get_raw_manifest(directory_ids: List[bytes], cur=None) → Iterable[Tuple[bytes, bytes]]

directory_get_id_range(start, end, limit=None, cur=None) → Iterator[Tuple[bytes]]

directory_get_random(cur=None)

revision_missing_from_list(revisions, cur=None)

revision_add_cols = ['id', 'date', 'date_offset', 'date_neg_utc_offset', 'date_offset_bytes', 'committer_date', 'committer_date_offset', 'committer_date_neg_utc_offset', 'committer_date_offset_bytes', 'type', 'directory', 'message', 'author_fullname', 'author_name', 'author_email', 'committer_fullname', 'committer_name', 'committer_email', 'metadata', 'synthetic', 'extra_headers', 'raw_manifest']

revision_get_cols = ['id', 'date', 'date_offset', 'date_neg_utc_offset', 'date_offset_bytes', 'committer_date', 'committer_date_offset', 'committer_date_neg_utc_offset', 'committer_date_offset_bytes', 'type', 'directory', 'message', 'author_fullname', 'author_name', 'author_email', 'committer_fullname', 'committer_name', 'committer_email', 'metadata', 'synthetic', 'extra_headers', 'raw_manifest', 'parents']

static mangle_query_key(key, main_table, id_col, ignore_displayname=False)

revision_get_from_list(revisions, ignore_displayname=False, cur=None)

revision_get_range(start, end, limit, cur=None) → Iterator[Tuple]

revision_log(root_revisions, ignore_displayname=False, limit=None, cur=None)

revision_shortlog_cols = ['id', 'parents']

revision_shortlog(root_revisions, limit=None, cur=None)

revision_get_random(cur=None)

extid_cols = ['extid', 'extid_version', 'extid_type', 'target', 'target_type']

extid_get_from_extid_list(extid_type: str, ids: List[bytes], version: Optional[int] = None, cur=None)

extid_get_from_swhid_list(target_type: str, ids: List[bytes], extid_version: Optional[int] = None, extid_type: Optional[str] = None, cur=None)

release_missing_from_list(releases, cur=None)

release_add_cols = ['id', 'target', 'target_type', 'date', 'date_offset', 'date_neg_utc_offset', 'date_offset_bytes', 'name', 'comment', 'synthetic', 'raw_manifest', 'author_fullname', 'author_name', 'author_email']

release_get_cols = ['id', 'target', 'target_type', 'date', 'date_offset', 'date_neg_utc_offset', 'date_offset_bytes', 'name', 'comment', 'synthetic', 'raw_manifest', 'author_fullname', 'author_name', 'author_email']

release_get_from_list(releases, ignore_displayname=False, cur=None)

release_get_range(start, end, limit, cur=None) → Iterator[Tuple]

release_get_random(cur=None)

snapshot_exists(snapshot_id, cur=None)

Check whether a snapshot with the given id exists

snapshot_missing_from_list(snapshots, cur=None)

snapshot_add(snapshot_id, cur=None)

Add a snapshot from the temporary table

snapshot_count_cols = ['target_type', 'count']

snapshot_count_branches(snapshot_id, branch_name_exclude_prefix=None, cur=None)

snapshot_get_cols = ['snapshot_id', 'name', 'target', 'target_type']

snapshot_get_by_id(snapshot_id, branches_from=b'', branches_count=None, target_types=None, branch_name_include_substring=None, branch_name_exclude_prefix=None, cur=None)

snapshot_get_id_range(start, end, limit=None, cur=None) → Iterator[Tuple[bytes]]

snapshot_get_random(cur=None)

origin_visit_add(origin, ts, type, cur=None)

Add a new origin_visit for origin origin at timestamp ts.

Parameters:

• origin – origin concerned by the visit

• ts – the date of the visit

• type – type of loader for the visit

Returns:

The new visit index step for that origin

origin_visit_status_cols = ['origin', 'visit', 'date', 'type', 'status', 'snapshot', 'metadata']

origin_visit_status_add(visit_status: OriginVisitStatus, cur=None) → None

Add new origin visit status

origin_visit_cols = ['origin', 'visit', 'date', 'type']

origin_visit_add_with_id(origin_visit: OriginVisit, cur=None) → None

Insert origin visit when id are already set

origin_visit_get_cols = ['origin', 'visit', 'date', 'type', 'status', 'metadata', 'snapshot']

origin_visit_select_cols = ['o.url AS origin', 'ov.visit', 'ov.date', 'ov.type AS type', 'ovs.status', 'ovs.snapshot', 'ovs.metadata']

origin_visit_status_select_cols = ['o.url AS origin', 'ovs.visit', 'ovs.date', 'ovs.type AS type', 'ovs.status', 'ovs.snapshot', 'ovs.metadata']

origin_visit_status_get_latest(origin_url: str, visit: int, allowed_statuses: Optional[List[str]] = None, require_snapshot: bool = False, cur=None) → Optional[Dict[str, Any]]

Given an origin visit id, return its latest origin_visit_status

origin_visit_status_get_range(origin: str, visit: int, date_from: Optional[datetime], order: ListOrder, limit: int, cur=None)

Retrieve visit_status rows for visit (origin, visit) in a paginated way.

origin_visit_get_range(origin: str, visit_from: int, order: ListOrder, limit: int, cur=None)

origin_visit_status_get_all_in_range(origin: str, allowed_statuses: Optional[List[str]], require_snapshot: bool, visit_from: int, visit_to: int, cur=None)

origin_visit_get(origin_id, visit_id, cur=None)

Retrieve information on visit visit_id of origin origin_id.

Parameters:

• origin_id – the origin concerned

• visit_id – The visit step for that origin

Returns:

The origin_visit information

origin_visit_find_by_date(origin, visit_date, cur=None)

origin_visit_exists(origin_id, visit_id, cur=None)

Check whether an origin visit with the given ids exists

origin_visit_get_latest(origin_id: str, type: Optional[str], allowed_statuses: Optional[Iterable[str]], require_snapshot: bool, cur=None)

Retrieve the most recent origin_visit of the given origin, with optional filters.

Parameters:

• origin_id – the origin concerned

• type – Optional visit type to filter on

• allowed_statuses – the visit statuses allowed for the returned visit

• require_snapshot (bool) – If True, only a visit with a known snapshot will be returned.

Returns:

The origin_visit information, or None if no visit matches.

origin_visit_get_random(type, cur=None)

Randomly select one origin visit that was full and in the last 3 months

origin_add(url, cur=None)

Insert a new origin and return the new identifier.

origin_cols = ['url']

origin_get_by_url(origins, cur=None)

Retrieve origin (type, url) from urls if found.

origin_get_by_sha1(sha1s, cur=None)

Retrieve origin urls from sha1s if found.

origin_id_get_by_url(origins, cur=None)

Retrieve origin (type, url) from urls if found.

origin_get_range_cols = ['id', 'url']

origin_get_range(origin_from: int = 1, origin_count: int = 100, cur=None)

Retrieve origin_count origins whose ids are greater or equal than origin_from.

Origins are sorted by id before retrieving them.

Parameters:

• origin_from – the minimum id of origins to retrieve

• origin_count – the maximum number of origins to retrieve

origin_search(url_pattern: str, offset: int = 0, limit: int = 50, regexp: bool = False, with_visit: bool = False, visit_types: Optional[List[str]] = None, cur=None)

Search for origins whose urls contain a provided string pattern or match a provided regular expression. The search is performed in a case insensitive way.

Parameters:

• url_pattern – the string pattern to search for in origin urls

• offset – number of found origins to skip before returning results

• limit – the maximum number of found origins to return

• regexp – if True, consider the provided pattern as a regular expression and returns origins whose urls match it

• with_visit – if True, filter out origins with no visit

origin_count(url_pattern, regexp=False, with_visit=False, cur=None)

Count origins whose urls contain a provided string pattern or match a provided regular expression. The pattern search in origin urls is performed in a case insensitive way.

Parameters:

• url_pattern (str) – the string pattern to search for in origin urls

• regexp (bool) – if True, consider the provided pattern as a regular expression and returns origins whose urls match it

• with_visit (bool) – if True, filter out origins with no visit

origin_snapshot_get_all(origin_url: str, cur=None) → Iterable[bytes]

object_find_by_sha1_git_cols = ['sha1_git', 'type']

object_find_by_sha1_git(ids, cur=None)

stat_counters(cur=None)

raw_extrinsic_metadata_get_cols = ['raw_extrinsic_metadata.target', 'raw_extrinsic_metadata.type', 'discovery_date', 'metadata_authority.type', 'metadata_authority.url', 'metadata_fetcher.id', 'metadata_fetcher.name', 'metadata_fetcher.version', 'origin', 'visit', 'snapshot', 'release', 'revision', 'path', 'directory', 'format', 'raw_extrinsic_metadata.metadata']

List of columns of the raw_extrinsic_metadata, metadata_authority, and metadata_fetcher tables, used when reading object metadata.

raw_extrinsic_metadata_add(id: bytes, type: str, target: str, discovery_date: datetime, authority_id: int, fetcher_id: int, format: str, metadata: bytes, origin: Optional[str], visit: Optional[int], snapshot: Optional[str], release: Optional[str], revision: Optional[str], path: Optional[bytes], directory: Optional[str], cur)

raw_extrinsic_metadata_get(target: str, authority_id: int, after_time: Optional[datetime], after_fetcher: Optional[int], limit: int, cur)

raw_extrinsic_metadata_get_by_ids(ids: List[bytes], cur=None)

raw_extrinsic_metadata_get_authorities(id: str, cur=None)

metadata_fetcher_cols = ['name', 'version']

metadata_fetcher_add(name: str, version: str, cur=None) → None

metadata_fetcher_get(name: str, version: str, cur=None)

metadata_fetcher_get_id(name: str, version: str, cur=None) → Optional[int]

metadata_authority_cols = ['type', 'url']

metadata_authority_add(type: str, url: str, cur=None) → None

metadata_authority_get(type: str, url: str, cur=None)

metadata_authority_get_id(type: str, url: str, cur=None) → Optional[int]

object_references_get(target_type: str, target: bytes, limit: int, cur=None)

object_references_add(reference_rows, cur=None) → None

object_references_create_partition(year: int, week: int, cur=None) → Tuple[date, date]

Create the partition of the object_references table for the given ISO year and week.