Source code for swh.objstorage.backends.winery.roshard

# Copyright (C) 2021-2022  The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information

import logging

import sh

from swh.perfecthash import Shard

from .throttler import Throttler

logger = logging.getLogger(__name__)


[docs]class Pool(object): name = "shards" def __init__(self, **kwargs): self.args = kwargs self.rbd = sh.sudo.bake("rbd", f"--pool={self.name}") self.ceph = sh.sudo.bake("ceph") self.image_size = int((self.args["shard_max_size"] * 2) / (1024 * 1024))
[docs] def image_list(self): try: self.rbd.ls() except sh.ErrorReturnCode_2 as e: if "No such file or directory" in e.args[0]: return [] else: raise return [image.strip() for image in self.rbd.ls()]
[docs] def image_path(self, image): return f"/dev/rbd/{self.name}/{image}"
[docs] def image_create(self, image): logger.info(f"rdb --pool {self.name} create --size={self.image_size} {image}") self.rbd.create( f"--size={self.image_size}", f"--data-pool={self.name}-data", image ) self.rbd.feature.disable( f"{self.name}/{image}", "object-map", "fast-diff", "deep-flatten" ) self.image_map(image, "rw")
[docs] def image_map(self, image, options): self.rbd.device("map", "-o", options, image) sh.sudo("chmod", "777", self.image_path(image))
[docs] def image_remap_ro(self, image): self.image_unmap(image) self.image_map(image, "ro")
[docs] def image_unmap(self, image): self.rbd.device.unmap(f"{self.name}/{image}", _ok_code=(0, 22))
[docs]class ROShard: def __init__(self, name, **kwargs): self.pool = Pool(shard_max_size=kwargs["shard_max_size"]) self.throttler = Throttler(**kwargs) self.name = name
[docs] def create(self, count): self.pool.image_create(self.name) self.shard = Shard(self.pool.image_path(self.name)) return self.shard.create(count)
[docs] def load(self): self.shard = Shard(self.pool.image_path(self.name)) return self.shard.load() == self.shard
[docs] def get(self, key): return self.throttler.throttle_get(self.shard.lookup, key)
[docs] def add(self, content, obj_id): return self.throttler.throttle_add(self.shard.write, obj_id, content)
[docs] def save(self): return self.shard.save()