swh.scheduler package

Subpackages:

• swh.scheduler.api package
  • swh.scheduler.api.client module
    • RemoteScheduler
  • swh.scheduler.api.serializers module
  • swh.scheduler.api.server module
    • get_global_scheduler()
    • SchedulerServerApp
    • argument_error_handler()
    • my_error_handler()
    • has_no_empty_params()
    • index()
    • site_map()
    • load_and_check_config()
    • make_app_from_configfile()
• swh.scheduler.celery_backend package
  • swh.scheduler.celery_backend.config module
    • setup_log_handler()
    • setup_queues_and_tasks()
    • on_worker_init()
    • monotonic()
    • route_for_task()
    • get_queue_stats()
    • get_queue_length()
    • get_available_slots()
    • register_task_class()
    • build_app()
  • swh.scheduler.celery_backend.pika_listener module
    • get_listener()
    • get_on_message()
    • process_event()
  • swh.scheduler.celery_backend.recurrent_visits module
    • DEFAULT_POLICY_CONFIG
    • MIN_SLOTS_RATIO
    • QUEUE_FULL_BACKOFF
    • DEFAULT_NO_ORIGINS_SCHEDULED_BACKOFF
    • BACKOFF_SPLAY
    • TERMINATE
    • grab_next_visits_policy_weights()
    • splay()
    • send_visits_for_visit_type()
    • visit_scheduler_thread()
    • VisitSchedulerThreads
    • spawn_visit_scheduler_thread()
    • terminate_visit_scheduler_threads()
  • swh.scheduler.celery_backend.runner module
    • run_ready_tasks()
    • main()
• swh.scheduler.cli package
  • swh.scheduler.cli.add_forge_now module
  • swh.scheduler.cli.admin module
  • swh.scheduler.cli.celery_monitor module
    • destination_from_pattern()
  • swh.scheduler.cli.journal module
  • swh.scheduler.cli.origin module
    • format_origins()
  • swh.scheduler.cli.simulator module
  • swh.scheduler.cli.task module
    • iter_origins()
  • swh.scheduler.cli.task_type module
    • ensure_task_type()
  • swh.scheduler.cli.test_cli_utils module
    • test_lister_task_type()
  • swh.scheduler.cli.utils module
    • schedule_origin_batches()
    • parse_argument()
    • parse_options()
    • get_task_type()
    • send_to_celery()
    • pretty_print_list()
    • pretty_print_dict()
    • format_dict()
    • pretty_print_run()
    • pretty_print_task()
    • task_add()
    • lister_task_type()
  • main()
• swh.scheduler.simulator package
  • swh.scheduler.simulator.common module
    • SimulationReport
    • Task
    • TaskEvent
    • Queue
    • Environment
  • swh.scheduler.simulator.origin_scheduler module
    • scheduler_runner_process()
    • scheduler_journal_client_process()
  • swh.scheduler.simulator.origins module
    • generate_listed_origin()
    • OriginModel
    • lister_process()
    • load_task_process()
  • swh.scheduler.simulator.task_scheduler module
    • scheduler_runner_process()
    • scheduler_listener_process()
  • update_metrics_process()
  • worker_process()
  • setup()
  • fill_test_data()
  • run()
• swh.scheduler.sql package
  • swh.scheduler.sql.upgrades package

Submodules:

• swh.scheduler.backend module
  • adapt_LastVisitStatus()
  • format_query()
  • SchedulerBackend
    • SchedulerBackend.current_version
    • SchedulerBackend.get_db()
    • SchedulerBackend.put_db()
    • SchedulerBackend.task_type_keys
    • SchedulerBackend.create_task_type()
    • SchedulerBackend.get_task_type()
    • SchedulerBackend.get_task_types()
    • SchedulerBackend.get_listers()
    • SchedulerBackend.get_listers_by_id()
    • SchedulerBackend.get_lister()
    • SchedulerBackend.get_or_create_lister()
    • SchedulerBackend.update_lister()
    • SchedulerBackend.record_listed_origins()
    • SchedulerBackend.get_listed_origins()
    • SchedulerBackend.grab_next_visits()
    • SchedulerBackend.task_create_keys
    • SchedulerBackend.task_keys
    • SchedulerBackend.create_tasks()
    • SchedulerBackend.set_status_tasks()
    • SchedulerBackend.disable_tasks()
    • SchedulerBackend.search_tasks()
    • SchedulerBackend.get_tasks()
    • SchedulerBackend.peek_ready_tasks()
    • SchedulerBackend.grab_ready_tasks()
    • SchedulerBackend.peek_ready_priority_tasks()
    • SchedulerBackend.grab_ready_priority_tasks()
    • SchedulerBackend.task_run_create_keys
    • SchedulerBackend.schedule_task_run()
    • SchedulerBackend.mass_schedule_task_runs()
    • SchedulerBackend.start_task_run()
    • SchedulerBackend.end_task_run()
    • SchedulerBackend.filter_task_to_archive()
    • SchedulerBackend.delete_archived_tasks()
    • SchedulerBackend.task_run_keys
    • SchedulerBackend.get_task_runs()
    • SchedulerBackend.origin_visit_stats_upsert()
    • SchedulerBackend.origin_visit_stats_get()
    • SchedulerBackend.visit_scheduler_queue_position_get()
    • SchedulerBackend.visit_scheduler_queue_position_set()
    • SchedulerBackend.update_metrics()
    • SchedulerBackend.get_metrics()
• swh.scheduler.cli_utils module
• swh.scheduler.exc module
  • SchedulerException
  • StaleData
  • UnknownPolicy
• swh.scheduler.interface module
  • PaginatedListedOriginList
    • PaginatedListedOriginList.results
  • SchedulerInterface
    • SchedulerInterface.create_task_type()
    • SchedulerInterface.get_task_type()
    • SchedulerInterface.get_task_types()
    • SchedulerInterface.create_tasks()
    • SchedulerInterface.set_status_tasks()
    • SchedulerInterface.disable_tasks()
    • SchedulerInterface.search_tasks()
    • SchedulerInterface.get_tasks()
    • SchedulerInterface.peek_ready_tasks()
    • SchedulerInterface.grab_ready_tasks()
    • SchedulerInterface.peek_ready_priority_tasks()
    • SchedulerInterface.grab_ready_priority_tasks()
    • SchedulerInterface.schedule_task_run()
    • SchedulerInterface.mass_schedule_task_runs()
    • SchedulerInterface.start_task_run()
    • SchedulerInterface.end_task_run()
    • SchedulerInterface.filter_task_to_archive()
    • SchedulerInterface.delete_archived_tasks()
    • SchedulerInterface.get_task_runs()
    • SchedulerInterface.get_listers()
    • SchedulerInterface.get_listers_by_id()
    • SchedulerInterface.get_lister()
    • SchedulerInterface.get_or_create_lister()
    • SchedulerInterface.update_lister()
    • SchedulerInterface.record_listed_origins()
    • SchedulerInterface.get_listed_origins()
    • SchedulerInterface.grab_next_visits()
    • SchedulerInterface.origin_visit_stats_upsert()
    • SchedulerInterface.origin_visit_stats_get()
    • SchedulerInterface.visit_scheduler_queue_position_get()
    • SchedulerInterface.visit_scheduler_queue_position_set()
    • SchedulerInterface.update_metrics()
    • SchedulerInterface.get_metrics()
• swh.scheduler.journal_client module
  • DISABLE_ORIGIN_THRESHOLD
  • MAX_NEXT_POSITION_OFFSET
  • max_date()
  • from_position_offset_to_days()
  • next_visit_queue_position()
  • get_last_status()
  • process_journal_objects()
• swh.scheduler.model module
  • check_timestamptz()
  • BaseSchedulerModel
    • BaseSchedulerModel.primary_key_columns()
    • BaseSchedulerModel.select_columns()
    • BaseSchedulerModel.insert_columns_and_metavars()
  • Lister
  • ListedOrigin
  • LastVisitStatus
    • LastVisitStatus.successful
    • LastVisitStatus.failed
    • LastVisitStatus.not_found
  • convert_last_visit_status()
  • OriginVisitStats
    • OriginVisitStats.url
    • OriginVisitStats.visit_type
    • OriginVisitStats.last_successful
    • OriginVisitStats.last_visit
    • OriginVisitStats.last_visit_status
    • OriginVisitStats.last_scheduled
    • OriginVisitStats.last_snapshot
    • OriginVisitStats.next_visit_queue_position
    • OriginVisitStats.next_position_offset
    • OriginVisitStats.successive_visits
    • OriginVisitStats.check_last_successful()
    • OriginVisitStats.check_last_visit()
  • SchedulerMetrics
    • SchedulerMetrics.lister_id
    • SchedulerMetrics.visit_type
    • SchedulerMetrics.last_update
    • SchedulerMetrics.origins_known
    • SchedulerMetrics.origins_enabled
    • SchedulerMetrics.origins_never_visited
    • SchedulerMetrics.origins_with_pending_changes
• swh.scheduler.pytest_plugin module
  • swh_scheduler_config()
  • swh_scheduler()
  • swh_db_scheduler()
  • swh_scheduler_celery_app()
  • swh_scheduler_celery_includes()
  • swh_scheduler_celery_worker()
• swh.scheduler.task module
  • ts()
  • SWHTask
    • SWHTask.reject_on_worker_lost
    • SWHTask.statsd
    • SWHTask.on_failure()
    • SWHTask.on_success()
    • SWHTask.log
    • SWHTask.ignore_result
    • SWHTask.priority
    • SWHTask.rate_limit
    • SWHTask.request_stack
    • SWHTask.run()
    • SWHTask.serializer
    • SWHTask.store_errors_even_if_ignored
    • SWHTask.track_started
    • SWHTask.typing
• swh.scheduler.utils module
  • utcnow()
  • get_task()
  • create_task_dict()
  • create_origin_task_dict()
  • create_origin_task_dicts()
  • create_oneshot_task_dict()

Module contents:

swh.scheduler.get_scheduler(cls: str, **kwargs) → SchedulerInterface

Get a scheduler object of class cls with arguments **kwargs.

Parameters:

• cls – scheduler’s class, either ‘local’ or ‘remote’

• kwargs – arguments to pass to the class’ constructor

Returns:

local: swh.scheduler.backend.SchedulerBackend remote: swh.scheduler.api.client.RemoteScheduler

Return type:

an instance of swh.scheduler, either local or remote

Raises:

ValueError if passed an unknown storage class. –

swh.scheduler.get_datastore(cls: str, **kwargs) → SchedulerInterface

Get a scheduler object of class cls with arguments **kwargs.

Parameters:

• cls – scheduler’s class, either ‘local’ or ‘remote’

• kwargs – arguments to pass to the class’ constructor

Returns:

local: swh.scheduler.backend.SchedulerBackend remote: swh.scheduler.api.client.RemoteScheduler

Return type:

an instance of swh.scheduler, either local or remote

Raises:

ValueError if passed an unknown storage class. –