swh.dataset package

Subpackages:

• swh.dataset.exporters package
  • swh.dataset.exporters.edges module
    • swhid()
    • GraphEdgesExporter
    • sort_graph_nodes()
  • swh.dataset.exporters.orc module
    • hash_to_hex_or_none()
    • swh_date_to_tuple()
    • datetime_to_tuple()
    • SWHTimestampConverter
    • ORCExporter
• swh.dataset.test namespace
  • swh.dataset.test.test_edges module
    • FakeDiskSet
    • exporter()
    • binhash()
    • hexhash()
    • b64e()
    • test_export_origin()
    • test_export_origin_visit_status()
    • test_export_snapshot_simple()
    • test_export_snapshot_aliases()
    • test_export_snapshot_no_pull_requests()
    • test_export_releases()
    • test_export_revision()
    • test_export_directory()
    • test_export_content()
    • zstwrite()
    • zstread()
    • test_sort_pipeline()
  • swh.dataset.test.test_orc module
    • orc_tmpdir()
    • orc_export()
    • orc_load()
    • exporter()
    • test_export_origin()
    • test_export_origin_visit()
    • test_export_origin_visit_status()
    • test_export_snapshot()
    • test_export_release()
    • test_export_revision()
    • test_export_directory()
    • test_export_content()
    • test_export_skipped_content()
    • test_date_to_tuple()
    • test_export_related_files()
    • test_export_related_files_separated()
    • test_export_invalid_max_rows()
    • test_export_content_with_data()
  • swh.dataset.test.test_utils module
    • diskset()
    • test_diskset()

Submodules:

• swh.dataset.athena module
  • create_database()
  • drop_table()
  • create_table()
  • repair_table()
  • query()
  • create_tables()
  • human_size()
  • run_query_get_results()
  • generate_subdataset()
• swh.dataset.cli module
  • run_export_graph()
• swh.dataset.exporter module
  • Exporter
    • Exporter.process_object()
    • Exporter.get_unique_file_id()
  • ExporterDispatch
    • ExporterDispatch.process_object()
• swh.dataset.journalprocessor module
  • JournalClientOffsetRanges
    • JournalClientOffsetRanges.subscribe()
    • JournalClientOffsetRanges.unsubscribe()
    • JournalClientOffsetRanges.process()
    • JournalClientOffsetRanges.handle_offset()
    • JournalClientOffsetRanges.deserialize_message()
    • JournalClientOffsetRanges.handle_messages()
  • ParallelJournalProcessor
    • ParallelJournalProcessor.get_offsets()
    • ParallelJournalProcessor.run()
    • ParallelJournalProcessor.progress_worker()
    • ParallelJournalProcessor.export_worker()
  • JournalProcessorWorker
    • JournalProcessorWorker.get_node_set_for_object()
    • JournalProcessorWorker.run()
    • JournalProcessorWorker.process_messages()
    • JournalProcessorWorker.process_message()
• swh.dataset.luigi module
  • Luigi tasks
    • File layout
    • Running all on staging
  • ObjectType
    • ObjectType.origin
    • ObjectType.origin_visit
    • ObjectType.origin_visit_status
    • ObjectType.snapshot
    • ObjectType.release
    • ObjectType.revision
    • ObjectType.directory
    • ObjectType.content
    • ObjectType.skipped_content
  • Format
    • Format.edges
    • Format.orc
  • merge_lists()
  • PathParameter
    • PathParameter.parse()
  • S3PathParameter
    • S3PathParameter.normalize()
  • FractionalFloatParameter
    • FractionalFloatParameter.parse()
  • stamps_paths()
  • ExportGraph
    • ExportGraph.config_file
    • ExportGraph.local_export_path
    • ExportGraph.export_id
    • ExportGraph.formats
    • ExportGraph.processes
    • ExportGraph.margin
    • ExportGraph.object_types
    • ExportGraph.output()
    • ExportGraph.complete()
    • ExportGraph.run()
  • UploadExportToS3
    • UploadExportToS3.local_export_path
    • UploadExportToS3.formats
    • UploadExportToS3.object_types
    • UploadExportToS3.s3_export_path
    • UploadExportToS3.requires()
    • UploadExportToS3.output()
    • UploadExportToS3.complete()
    • UploadExportToS3.run()
  • DownloadExportFromS3
    • DownloadExportFromS3.local_export_path
    • DownloadExportFromS3.formats
    • DownloadExportFromS3.object_types
    • DownloadExportFromS3.s3_export_path
    • DownloadExportFromS3.parallelism
    • DownloadExportFromS3.requires()
    • DownloadExportFromS3.output()
    • DownloadExportFromS3.complete()
    • DownloadExportFromS3.run()
  • LocalExport
    • LocalExport.local_export_path
    • LocalExport.formats
    • LocalExport.object_types
    • LocalExport.export_task_type
    • LocalExport.requires()
    • LocalExport.output()
    • LocalExport.complete()
  • AthenaDatabaseTarget
    • AthenaDatabaseTarget.exists()
  • CreateAthena
    • CreateAthena.object_types
    • CreateAthena.s3_export_path
    • CreateAthena.s3_athena_output_location
    • CreateAthena.athena_db_name
    • CreateAthena.requires()
    • CreateAthena.output()
    • CreateAthena.run()
  • RunExportAll
    • RunExportAll.formats
    • RunExportAll.object_types
    • RunExportAll.s3_export_path
    • RunExportAll.s3_athena_output_location
    • RunExportAll.athena_db_name
    • RunExportAll.requires()
• swh.dataset.relational module
• swh.dataset.utils module
  • ZSTFile
    • ZSTFile.read()
    • ZSTFile.write()
  • SQLiteSet
    • SQLiteSet.add()
  • LevelDBSet
    • LevelDBSet.add()
  • remove_pull_requests()

Module contents: