swh.objstorage.backends.winery package

Subpackages

• swh.objstorage.backends.winery.sql package
  • Module contents

Submodules

• swh.objstorage.backends.winery.database module
  • Database
    • Database.pool
    • Database.list_shard_tables()
• swh.objstorage.backends.winery.gunicorn module
  • worker_exit()
  • worker_int()
  • shutdown_storage_backend()
  • ThrottledAccessLog
    • ThrottledAccessLog.filter()
• swh.objstorage.backends.winery.objstorage module
  • WineryObjStorage
    • WineryObjStorage.PRIMARY_HASH
    • WineryObjStorage.name
    • WineryObjStorage.get()
    • WineryObjStorage.check_config()
    • WineryObjStorage.add()
    • WineryObjStorage.delete()
    • WineryObjStorage.list_content()
    • WineryObjStorage.on_shutdown()
  • WineryReader
    • WineryReader.list_signatures()
    • WineryReader.roshard()
    • WineryReader.rwshard()
    • WineryReader.get()
    • WineryReader.on_shutdown()
  • pack()
  • cleanup_rw_shard()
  • WineryWriter
    • WineryWriter.release_shard()
    • WineryWriter.shard
    • WineryWriter.add()
    • WineryWriter.delete()
    • WineryWriter.check()
    • WineryWriter.pack()
    • WineryWriter.on_shutdown()
  • never_stop()
  • stop_after_shards()
  • shard_packer()
  • rw_shard_cleaner()
  • deleted_objects_cleaner()
• swh.objstorage.backends.winery.roshard module
  • ShardNotMapped
  • Pool
    • Pool.image_exists()
    • Pool.image_mapped()
    • Pool.image_list()
    • Pool.image_path()
    • Pool.image_create()
    • Pool.image_map()
    • Pool.image_unmap()
    • Pool.image_remap_ro()
  • RBDPool
    • RBDPool.POOL_CONFIG
    • RBDPool.from_kwargs()
    • RBDPool.run()
    • RBDPool.rbd()
    • RBDPool.image_exists()
    • RBDPool.image_list()
    • RBDPool.image_path()
    • RBDPool.image_create()
    • RBDPool.image_map()
    • RBDPool.image_unmap()
  • FileBackedPool
    • FileBackedPool.image_exists()
    • FileBackedPool.image_list()
    • FileBackedPool.image_path()
    • FileBackedPool.image_create()
    • FileBackedPool.image_map()
    • FileBackedPool.image_unmap()
    • FileBackedPool.image_unmap_all()
  • pool_from_settings()
  • record_shard_mapped()
  • manage_images()
  • ROShard
    • ROShard.open()
    • ROShard.get()
    • ROShard.close()
    • ROShard.delete()
  • ROShardCreator
    • ROShardCreator.zero_image_if_needed()
    • ROShardCreator.add()
• swh.objstorage.backends.winery.rwshard module
  • IdleHandler
    • IdleHandler.quiesce()
    • IdleHandler.reset()
    • IdleHandler.quiesce_then_reset()
    • IdleHandler.join()
    • IdleHandler.run()
  • ShardIdleTimeoutCallback
  • RWShard
    • RWShard.disable_idle_handler()
    • RWShard.name
    • RWShard.table_name
    • RWShard.is_full()
    • RWShard.create()
    • RWShard.drop()
    • RWShard.total_size()
    • RWShard.add()
    • RWShard.get()
    • RWShard.delete()
    • RWShard.all()
    • RWShard.count()
• swh.objstorage.backends.winery.settings module
  • Packer
    • Packer.create_images
    • Packer.pack_immediately
    • Packer.clean_immediately
  • packer_settings_with_defaults()
  • Shards
    • Shards.max_size
    • Shards.rw_idle_timeout
  • shards_settings_with_defaults()
  • ShardsPool
    • ShardsPool.type
  • RbdShardsPool
    • RbdShardsPool.use_sudo
    • RbdShardsPool.map_options
    • RbdShardsPool.pool_name
    • RbdShardsPool.data_pool_name
    • RbdShardsPool.image_features_unsupported
    • RbdShardsPool.type
  • rbd_shards_pool_settings_with_defaults()
  • DirectoryShardsPool
    • DirectoryShardsPool.base_directory
    • DirectoryShardsPool.pool_name
    • DirectoryShardsPool.type
  • directory_shards_pool_settings_with_defaults()
  • Throttler
    • Throttler.db
    • Throttler.max_read_bps
    • Throttler.max_write_bps
  • Database
    • Database.db
    • Database.application_name
  • database_settings_with_defaults()
  • Winery
    • Winery.database
    • Winery.shards
    • Winery.shards_pool
    • Winery.throttler
    • Winery.packer
  • populate_default_settings()
• swh.objstorage.backends.winery.sharedbase module
  • ShardState
    • ShardState.STANDBY
    • ShardState.WRITING
    • ShardState.FULL
    • ShardState.PACKING
    • ShardState.PACKED
    • ShardState.CLEANING
    • ShardState.READONLY
    • ShardState.locked
    • ShardState.image_available
    • ShardState.readonly
  • SignatureState
    • SignatureState.INFLIGHT
    • SignatureState.PRESENT
    • SignatureState.DELETED
  • TemporaryShardLocker
  • SharedBase
    • SharedBase.current_version
    • SharedBase.locked_shard
    • SharedBase.locked_shard_id
    • SharedBase.set_locked_shard()
    • SharedBase.lock_one_shard()
    • SharedBase.maybe_lock_one_shard()
    • SharedBase.set_shard_state()
    • SharedBase.create_shard()
    • SharedBase.shard_packing_starts()
    • SharedBase.shard_packing_ends()
    • SharedBase.get_shard_info()
    • SharedBase.get_shard_state()
    • SharedBase.list_shards()
    • SharedBase.count_objects()
    • SharedBase.record_shard_mapped()
    • SharedBase.contains()
    • SharedBase.get()
    • SharedBase.record_new_obj_id()
    • SharedBase.list_signatures()
    • SharedBase.delete()
    • SharedBase.deleted_objects()
    • SharedBase.clean_deleted_object()
• swh.objstorage.backends.winery.sleep module
  • sleep_exponential()
• swh.objstorage.backends.winery.throttler module
  • LeakyBucket
    • LeakyBucket.reset()
    • LeakyBucket.add()
  • BandwidthCalculator
    • BandwidthCalculator.add()
    • BandwidthCalculator.get()
  • IOThrottler
    • IOThrottler.init_db()
    • IOThrottler.download_info()
    • IOThrottler.upload_info()
    • IOThrottler.add()
    • IOThrottler.sync()
    • IOThrottler.maybe_sync()
  • Throttler
    • Throttler.from_settings()
    • Throttler.throttle_get()
    • Throttler.throttle_add()
  • NoopThrottler
    • NoopThrottler.throttle_get()
    • NoopThrottler.throttle_add()

Module contents

class swh.objstorage.backends.winery.WineryObjStorage(database: Database, shards: Shards, shards_pool: ShardsPool, throttler: Throttler, packer: Packer | None = None, readonly: bool = False, allow_delete: bool = False, name: str = 'winery')

Bases: ObjStorage

PRIMARY_HASH: LiteralPrimaryHash = 'sha256'

name: str = 'winery'

Default objstorage name; can be overloaded at instantiation time giving a ‘name’ argument to the constructor

get(obj_id: ObjId) → bytes

Retrieve the content of a given object.

Parameters:

obj_id – object id.

Returns:

the content of the requested object as bytes.

Raises:

ObjNotFoundError – if the requested object is missing.

check_config(*, check_write: bool) → bool

Check whether the object storage is properly configured.

Parameters:

• check_write (bool) – if True, check if writes to the object storage

• succeed. (can)

Returns:

True if the configuration check worked, False if ‘check_write’ is True and the object storage is actually read only, and an exception if the check failed.

add(content: bytes, obj_id: ObjId, check_presence: bool = True) → None

Add a new object to the object storage.

Parameters:

• content – object’s raw content to add in storage.

• obj_id – either dict of checksums, or single checksum of [bytes] using [ID_HASH_ALGO] algorithm. It is trusted to match the bytes.

• check_presence (bool) – indicate if the presence of the content should be verified before adding the file.

Returns:

the id (bytes) of the object into the storage.

delete(obj_id: ObjId)

Delete an object.

Parameters:

obj_id – object identifier.

Raises:

ObjNotFoundError – if the requested object is missing.

list_content(last_obj_id: ObjId | None = None, limit: int | None = 10000) → Iterator[ObjId]

Generates known object ids.

Parameters:

• last_obj_id – object id from which to iterate from (excluded).

• limit (int) – max number of object ids to generate. If unset (None), generate all objects (behavior might not be guaranteed for all backends).

Generates:

obj_id: object ids.

on_shutdown()

swh.objstorage.backends.winery.get_datastore(**cfg)