Source code for

# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information

from typing import Callable

from swh.counters import get_counters
from swh.counters.interface import CountersInterface
from import StorageSpec, get_storage
from import StorageInterface


[docs] class CountingProxyStorage: """Counting Storage Proxy. This is in charge of adding objects directly to swh-counters, without going through Kafka/swh-journal. This is meant as a simple way to setup counters for experiments; production should use swh-journal to reduce load/latency of the storage server. Additionally, unlike the journal-based counting, it does not count persons or the number of origins per netloc. Sample configuration use case for filtering storage: .. code-block: yaml storage: cls: counter counters: cls: remote url: storage: cls: remote url: """ def __init__(self, counters, storage: StorageSpec) -> None: self.counters: CountersInterface = get_counters(**counters) StorageInterface = get_storage(**storage) def __getattr__(self, key): if key == "storage": raise AttributeError(key) if key.endswith("_add"): return self._adder(key[0:-4], getattr(, key)) return getattr(, key) def _adder(self, collection: str, backend_function: Callable): def f(objs): self.counters.add(collection, [obj.unique_key() for obj in objs]) return backend_function(objs) return f