{ "title": "Source structure and extraction atlas", "version": "2026-09-11-atlas", "doi_status": "reserved; unpublished", "doi": "10.5281/zenodo.22693083", "selection": "61 catalogue entries: 53 registered sources, 5 constituent score collections, 3 audit-only entries", "code_revision": "2bb6e85282e3ab11af1bcc7126290f67af176f53", "code_inspection_method": "Static Python syntax inspection. Effective methods resolve same-class redefinitions and local inheritance. Literal get calls and dictionary entries are syntactic observations, not inferred native schemas or end-to-end mappings. Helpers, dynamic dispatch and imported normalizers require separate examination. No application code was executed.", "scope_limit": "Static implementation inspection and previously dated source observations; no new live-source census, extraction-accuracy validation or full runtime replay. Expressions are code evidence, not automatic source-to-target crosswalks.", "field_guide": { "atlas_editorial_reading": "Authored code reading and interpretation; native observations are kept separately in selected_document.", "implementation.configuration": "Effective literal class defaults with owning class and code locator; runtime overrides were not evaluated.", "implementation.methods": "Effective selected acquisition/domain methods with syntactically observed get calls, HTML queries and dictionary constructions.", "implementation.methods.dictionary_entries": "Local dictionary key and Python expression. This does not establish the final output path, data-flow origin, or execution on a particular record.", "implementation.methods.literal_get_calls": "Literal key read from the named expression. The receiver may be an upstream object, temporary object, configuration or normalized output.", "shared_helper_modules": "Separately indexed helpers and normalizers; availability does not assert that a particular frozen record passed through them.", "selected_document": "Previously documented observation of the cited page on its stated date, not a representative collection sample." }, "records": [ { "catalogue_id": "S001", "source_key": "aeolianskinner", "name": "Aeolian-Skinner Archive", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Steve Layden (website creator); Jeff Scofield (website manager)", "scope_description": "Skinner and Aeolian-Skinner instruments", "documentary_unit": "Factory specification or manufacturer revision", "source_description": "The archive presents stoplists, photographs and recollections arranged around the firm's instruments. Specifications describe delivery from the factory and revisions by Skinner or Aeolian-Skinner; subsequent alterations by other builders are excluded.", "source_description_references": [ "aeolianskinner_main", "record_aeolianskinner" ], "interpretation": "An opus number can join this archive to an instrument register, but the specifications must be attached to the manufacturer's documented configuration. They cannot stand for the instrument's present state.", "rights_observation": "The archive displays its own copyright notice; no general open licence was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "aeolianskinner_main", "record_aeolianskinner" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_aeolianskinner", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Static opus page with identification block, specification and documentary notes", "repeated_information": "Division columns contain stop name, pitch and pipe quantity or borrowing reference.", "date_and_state_distinctions": "The specification represents the manufacturer state. Later alterations appear in a separate narrative note.", "interpretive_limit": "A stoplist and a later note can concern different states of the same instrument.", "observations": [ "Town, state and church head the record.", "Named manufacturer and opus number.", "Delivery year and a separately dated later alteration.", "Division-based stops include pitches and borrowing references.", "Separate Great, Swell, Choir, Echo and Pedal groups.", "The later note describes replacement of the console.", "Photograph and named photo supplier.", "A named specification source is supplied.", "Cross-reference to an OHS instrument record.", "Specification and photograph have separate source credits." ], "uninspected_or_unfilled_material": [], "narrative": "Opus 511 is presented through a factory specification, division columns and documentary notes. Stop entries include pitches, pipe quantities and borrowing references. A later note concerns replacement of the console. The specification and that alteration note therefore describe different documentary states; the date of the later note cannot be assigned to the factory stoplist." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The archive subclass has dedicated index, title and specification extraction for opus pages. Its source scope is the manufacturer configuration described by the archive, subject to the limitations stated in the cited source account.", "transformation": "The specialised stoplist routine handles division-oriented specification material, while the shared transform supplies other metadata, raw text, media and source-qualified identity. Pipe quantities and borrowing annotations in the observed page belong to individual stop entries and the stated factory configuration.", "limitations": "The inspected source may also contain later documentary notes. A note about a replacement console does not redate the factory stoplist. Neither the generic builder/year projection nor a matched opus supplies independent evidence of the current configuration." }, "implementation": { "class": "AeolianSkinnerArchiveCrawler", "class_line": 1172, "path": "aggregator/static_html_crawler.py", "inheritance": [ "AeolianSkinnerArchiveCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "aeolianskinner", "path": "aggregator/static_html_crawler.py", "line": 1173, "class": "AeolianSkinnerArchiveCrawler" }, "id_column": { "value": "aeolianskinner_id", "path": "aggregator/static_html_crawler.py", "line": 1174, "class": "AeolianSkinnerArchiveCrawler" }, "default_country": { "value": "United States", "path": "aggregator/static_html_crawler.py", "line": 1177, "class": "AeolianSkinnerArchiveCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://archive.pipeorgandatabase.org/aeolianskinner/aeolian_skinner.html", "https://archive.pipeorgandatabase.org/aeolianskinner/skinner.html", "https://archive.pipeorgandatabase.org/aeolianskinner/e_m_skinner_son.html", "https://archive.pipeorgandatabase.org/aeolianskinner/bassett_skinner.html" ], "path": "aggregator/static_html_crawler.py", "line": 1178, "class": "AeolianSkinnerArchiveCrawler" }, "detail_url_patterns": { "value": [ "/aeolianskinner/specs/op" ], "path": "aggregator/static_html_crawler.py", "line": 1184, "class": "AeolianSkinnerArchiveCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://archive.pipeorgandatabase.org/aeolianskinner", "path": "aggregator/static_html_crawler.py", "line": 1176, "class": "AeolianSkinnerArchiveCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "AeolianSkinnerArchiveCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 1222, "line_end": 1266, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_looks_like_aeolian_stop" ], "literal_get_calls": [], "html_queries": [ { "line": 1227, "expression": "soup.find_all('pre')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "AeolianSkinnerArchiveCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 1206, "line_end": 1209, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "AeolianSkinnerArchiveCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 1211, "line_end": 1220, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 1212, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "AeolianSkinnerArchiveCrawler._looks_like_aeolian_stop", "path": "aggregator/static_html_crawler.py", "line_start": 1269, "line_end": 1271, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "AeolianSkinnerArchiveCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 1186, "line_end": 1204, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_reference_from_parts" ], "literal_get_calls": [], "html_queries": [ { "line": 1189, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 1201, "key": "opus", "value_expression": "opus" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 434, "line_end": 516, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_extract_images", "_extract_links", "_extract_metadata", "_extract_raw_spec", "_extract_table_rows", "_extract_title", "_first_int", "_first_metadata", "_first_year", "_location_parts", "_main_text", "_safe_json_loads" ], "literal_get_calls": [ { "line": 435, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 440, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 441, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 442, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 443, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 444, "receiver": "raw_data", "key": "rank_count", "expression": "raw_data.get('rank_count')" }, { "line": 513, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 442, "receiver": "metadata", "key": "manuals", "expression": "metadata.get('manuals')" }, { "line": 442, "receiver": "metadata", "key": "manual", "expression": "metadata.get('manual')" }, { "line": 443, "receiver": "metadata", "key": "stops", "expression": "metadata.get('stops')" }, { "line": 443, "receiver": "metadata", "key": "stop", "expression": "metadata.get('stop')" }, { "line": 444, "receiver": "metadata", "key": "ranks", "expression": "metadata.get('ranks')" }, { "line": 444, "receiver": "metadata", "key": "rank", "expression": "metadata.get('rank')" }, { "line": 480, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 453, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 454, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 479, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 487, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': raw_data.get('country') or self.default_country, 'region': raw_data.get('region') or self.default_region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 460, "key": "involved", "value_expression": "self._build_involved(builder, year)" }, { "line": 461, "key": "events", "value_expression": "[{'type': 'construction', 'date': str(year), 'description': builder}] if year else []" }, { "line": 462, "key": "specifications", "value_expression": "{'manual_count': manual_count, 'stop_count': stop_count, 'rank_count': rank_count, 'stops_by_division': stops_by_division, 'stops': [{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops], 'raw_spec': raw_spec, 'technical_rows_unstructured': [{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows}" }, { "line": 479, "key": "media", "value_expression": "{'images': self._extract_images(soup, raw_data.get('url') or '')}" }, { "line": 480, "key": "links", "value_expression": "self._extract_links(soup, raw_data.get('url') or '')" }, { "line": 481, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': text}] if text else []" }, { "line": 482, "key": "other_identifiers", "value_expression": "[{'type': self.id_column, 'value': raw_data.get(self.id_column)}]" }, { "line": 483, "key": "extensions", "value_expression": "{self.source_key: {'title': title, 'metadata': metadata, 'raw_data': self._safe_json_loads(raw_data.get('raw_data_json'), {}), 'reference_context': {key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}}}" }, { "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 467, "key": "stops", "value_expression": "[{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 472, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 473, "key": "technical_rows_unstructured", "value_expression": "[{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows" }, { "line": 479, "key": "images", "value_expression": "self._extract_images(soup, raw_data.get('url') or '')" }, { "line": 452, "key": "status", "value_expression": "'current_location'" }, { "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "line": 454, "key": "region", "value_expression": "raw_data.get('region') or self.default_region" }, { "line": 455, "key": "town", "value_expression": "town" }, { "line": 456, "key": "building", "value_expression": "building" }, { "line": 457, "key": "raw_str", "value_expression": "title" }, { "line": 482, "key": "type", "value_expression": "self.id_column" }, { "line": 482, "key": "value", "value_expression": "raw_data.get(self.id_column)" }, { "line": 485, "key": "title", "value_expression": "title" }, { "line": 486, "key": "metadata", "value_expression": "metadata" }, { "line": 487, "key": "raw_data", "value_expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {})" }, { "line": 488, "key": "reference_context", "value_expression": "{key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}" }, { "line": 461, "key": "type", "value_expression": "'construction'" }, { "line": 461, "key": "date", "value_expression": "str(year)" }, { "line": 461, "key": "description", "value_expression": "builder" }, { "line": 468, "key": "name", "value_expression": "stop" }, { "line": 468, "key": "division", "value_expression": "division" }, { "line": 468, "key": "raw_str", "value_expression": "stop" }, { "line": 481, "key": "type", "value_expression": "'source_text'" }, { "line": 481, "key": "text", "value_expression": "text" }, { "line": 474, "key": "label", "value_expression": "key" }, { "line": 474, "key": "value", "value_expression": "value" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 457, "key": "raw_str", "value_expression": "title" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 461, "line_end": 469, "text": " \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 460, "line_end": 468, "text": " \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 462, "line_end": 470, "text": " \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 463, "line_end": 471, "text": " \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 450, "line_end": 458, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": raw_data.get(\"country\") or self.default_country,\n \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 454, "line_end": 462, "text": " \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }\n ],\n \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {" } ] } }, { "catalogue_id": "S002", "source_key": "arpschnitger", "name": "Arp Schnitger Organs", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Geert Jan Pottjewijd", "scope_description": "Arp Schnitger organs; related material on Drenthe", "documentary_unit": "Builder-focused organ profile", "source_description": "This personal site documents Schnitger organs and also carries material on organs in Drenthe. Its entry page names Geert Jan Pottjewijd and gives 4 April 2026 as its last update. Historical announcements remain on the same page.", "source_description_references": [ "arpschnitger", "record_arpschnitger" ], "interpretation": "The builder-centred selection makes this a useful specialist counterpart to regional inventories. The date on the home page does not date every instrument description, and the different sections should not be counted as one geographical census.", "rights_observation": "No general reuse licence was established from the consulted entry page.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "arpschnitger", "record_arpschnitger" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_arpschnitger", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Static historical essay with photographs and a later specification", "repeated_information": "Dated narrative entries and a division-based stop table.", "date_and_state_distinctions": "One page follows a case across replacement instruments and relocation to another church.", "interpretive_limit": "A page is not necessarily one instrument at one place. The later stoplist belongs to the organ installed in the transferred case.", "observations": [ "Glückstadt and the subsequent destination of the case are named.", "Several builders are associated with dated interventions.", "Chronological entries distinguish construction, replacement and relocation.", "Later specification arranged by divisions and pitches.", "Hauptwerk, Rückpositiv and Pedal are separate groups.", "Case survival and transfer are described.", "The original organ is identified as former; the case survives elsewhere.", "Historical and later case photographs with credits.", "Named information provider and an image-source citation.", "Links to the later case photograph and its Wikimedia source.", "Information and photograph contributors are named." ], "uninspected_or_unfilled_material": [], "narrative": "The Glückstadt account follows the survival and transfer of a case into another instrument. Dated narrative entries concern successive builders and locations, while the later specification belongs to an organ in the transferred case. The page thus supplies evidence about material continuity that would be lost if its URL were treated as one unchanged instrument at one address." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Configured index and detail-link patterns identify the specialist site's organ pages. The subclass supplies page-title and location conventions.", "transformation": "The transform delegates its substantive extraction to the common static-page implementation. The distinction matters: a named source class does not mean that its historical narratives have a dedicated event parser. Raw text, metadata, table material and images are preserved alongside the common summary projection.", "limitations": "A case can move to another location and become part of another instrument. The observed source account documents precisely this problem. A single URL, current-location label or first year cannot establish uninterrupted instrument identity or one configuration." }, "implementation": { "class": "ArpSchnitgerCrawler", "class_line": 1339, "path": "aggregator/static_html_crawler.py", "inheritance": [ "ArpSchnitgerCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "arpschnitger", "path": "aggregator/static_html_crawler.py", "line": 1340, "class": "ArpSchnitgerCrawler" }, "id_column": { "value": "arpschnitger_id", "path": "aggregator/static_html_crawler.py", "line": 1341, "class": "ArpSchnitgerCrawler" }, "default_country": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 28, "class": "StaticHtmlOrganCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://arpschnitger.nl/schnit.html" ], "path": "aggregator/static_html_crawler.py", "line": 1344, "class": "ArpSchnitgerCrawler" }, "detail_url_patterns": { "value": [ "arpschnitger.nl/s" ], "path": "aggregator/static_html_crawler.py", "line": 1345, "class": "ArpSchnitgerCrawler" }, "exclude_url_patterns": { "value": [ "schnit.html", "schnintro", "sdiscogr", "schnitlit", "schnitdvd", "schnittek", "youtube", ".pdf", ".wmv" ], "path": "aggregator/static_html_crawler.py", "line": 1346, "class": "ArpSchnitgerCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://arpschnitger.nl", "path": "aggregator/static_html_crawler.py", "line": 1343, "class": "ArpSchnitgerCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "ArpSchnitgerCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 1392, "line_end": 1396, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 1393, "expression": "soup.find(['b', 'strong'])" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "ArpSchnitgerCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 1358, "line_end": 1377, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding" ], "literal_get_calls": [ { "line": 1367, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "ArpSchnitgerCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 1401, "line_end": 1408, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "ArpSchnitgerCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 1379, "line_end": 1390, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts" ], "literal_get_calls": [], "html_queries": [ { "line": 1382, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "ArpSchnitgerCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 1398, "line_end": 1399, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 1399, "key": "builder", "value_expression": "'Arp Schnitger'" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 79, "kind": "input read" }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "key": "max_items", "value_expression": "kwargs.get('max_items')", "line": 80, "kind": "input read" }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "key": "start_urls", "value_expression": "kwargs.get('start_urls')", "line": 81, "kind": "input read" }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "key": "rate_limit_delay", "value_expression": "self.config.get('rate_limit_delay', 0.2)", "line": 117, "kind": "input read" }, { "symbol": "ArpSchnitgerCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "key": "timeout", "value_expression": "self.config.get('timeout', 30)", "line": 1367, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.crawl", "line_start": 78, "line_end": 84, "text": " def crawl(self, **kwargs) -> List[Dict[str, Any]]:\n task_id = kwargs.get(\"task_id\") or f\"{self.source_key}_crawl_{datetime.now().strftime('%Y%m%d%H%M%S')}\"\n max_items = self._optional_int(kwargs.get(\"max_items\"))\n start_urls = kwargs.get(\"start_urls\") or self.start_urls\n if isinstance(start_urls, str):\n start_urls = [item.strip() for item in start_urls.split(\",\") if item.strip()]\n" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.crawl", "line_start": 78, "line_end": 85, "text": " def crawl(self, **kwargs) -> List[Dict[str, Any]]:\n task_id = kwargs.get(\"task_id\") or f\"{self.source_key}_crawl_{datetime.now().strftime('%Y%m%d%H%M%S')}\"\n max_items = self._optional_int(kwargs.get(\"max_items\"))\n start_urls = kwargs.get(\"start_urls\") or self.start_urls\n if isinstance(start_urls, str):\n start_urls = [item.strip() for item in start_urls.split(\",\") if item.strip()]\n\n self._update_scrape_status(task_id, \"INITIALIZING\", f\"Starting {self.display_name} crawl...\", 0, 0)" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.crawl", "line_start": 78, "line_end": 86, "text": " def crawl(self, **kwargs) -> List[Dict[str, Any]]:\n task_id = kwargs.get(\"task_id\") or f\"{self.source_key}_crawl_{datetime.now().strftime('%Y%m%d%H%M%S')}\"\n max_items = self._optional_int(kwargs.get(\"max_items\"))\n start_urls = kwargs.get(\"start_urls\") or self.start_urls\n if isinstance(start_urls, str):\n start_urls = [item.strip() for item in start_urls.split(\",\") if item.strip()]\n\n self._update_scrape_status(task_id, \"INITIALIZING\", f\"Starting {self.display_name} crawl...\", 0, 0)\n refs: List[Dict[str, Any]] = []" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.crawl", "line_start": 114, "line_end": 122, "text": " break\n if max_items and len(refs) >= max_items:\n break\n time.sleep(get_jittered_sleep_duration(self.config.get(\"rate_limit_delay\", 0.2)))\n\n self._update_scrape_status(\n task_id,\n \"COMPLETED\",\n f\"{self.display_name} crawl completed. Upserted {len(refs)} references.\"," }, { "path": "aggregator/static_html_crawler.py", "symbol": "ArpSchnitgerCrawler._fetch_html", "line_start": 1364, "line_end": 1372, "text": " \"\"\"\n response = self.session.get(\n url,\n timeout=int(self.config.get(\"timeout\", 30)),\n verify=False,\n )\n response.raise_for_status()\n hostname = (urlparse(response.url).hostname or \"\").lower()\n if hostname not in {\"arpschnitger.nl\", \"www.arpschnitger.nl\"}:" } ] } }, { "catalogue_id": "S003", "source_key": "augustine_virtual_organs_samples", "name": "Augustine's Virtual Organs Sample Set Listings", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Augustine's Virtual Organs", "scope_description": "Hauptwerk products grouped by style, builder and manual count", "documentary_unit": "Original sample-set listing", "source_description": "The catalogue separates original recordings from composite sets and marks a subset as free. It also distinguishes the producer's own recordings from recordings supplied by others and subsequently made into sample sets.", "source_description_references": [ "augustine_virtual_organs_samples" ], "interpretation": "Original, composite and derived products need separate records. The free category describes distribution price; it does not supply a common licence for the audio.", "rights_observation": "Copyright is asserted in the footer; product-specific permissions remain to be checked.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "augustine_virtual_organs_samples" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects links to PHP product pages and their surrounding context. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is sample-set entries. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Original, composite and derived products need separate provenance. A composite cannot stand as a measured historical organ configuration. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "AugustineVirtualOrgansSamplesCrawler", "class_line": 1399, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "AugustineVirtualOrgansSamplesCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "augustine_virtual_organs_samples", "path": "aggregator/grandorgue_library_crawler.py", "line": 1400, "class": "AugustineVirtualOrgansSamplesCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_registry", "path": "aggregator/grandorgue_library_crawler.py", "line": 1402, "class": "AugustineVirtualOrgansSamplesCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1404, "line_end": 1466, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1429, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 1425, "expression": "soup.select('a[href*=\".php\"]')" } ], "dictionary_entries": [ { "line": 1438, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Sample sets'}" }, { "line": 1439, "key": "item", "value_expression": "{'title': title, 'producer_label': \"Augustine's Virtual Organs\", 'description_excerpt': f\"Public Augustine's Virtual Organs listing page for {title}.\", 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'free_or_low_cost' if 'free' in lower else 'producer_catalog', 'availability_status_hint': 'producer_listing'}" }, { "line": 1448, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'producer_listing', 'is_odf_only': False, 'requires_sample_set': False, 'is_demo': False}" }, { "line": 1455, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1456, "key": "extensions", "value_expression": "{'source_fragment_html': str(link)}" }, { "line": 1438, "key": "position", "value_expression": "position" }, { "line": 1438, "key": "section_label", "value_expression": "'Sample sets'" }, { "line": 1440, "key": "title", "value_expression": "title" }, { "line": 1441, "key": "producer_label", "value_expression": "\"Augustine's Virtual Organs\"" }, { "line": 1442, "key": "description_excerpt", "value_expression": "f\"Public Augustine's Virtual Organs listing page for {title}.\"" }, { "line": 1443, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1444, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1445, "key": "access_model_hint", "value_expression": "'free_or_low_cost' if 'free' in lower else 'producer_catalog'" }, { "line": 1446, "key": "availability_status_hint", "value_expression": "'producer_listing'" }, { "line": 1449, "key": "package_title", "value_expression": "title" }, { "line": 1450, "key": "package_kind", "value_expression": "'producer_listing'" }, { "line": 1451, "key": "is_odf_only", "value_expression": "False" }, { "line": 1452, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1453, "key": "is_demo", "value_expression": "False" }, { "line": 1456, "key": "source_fragment_html", "value_expression": "str(link)" }, { "line": 1455, "key": "url", "value_expression": "detail_url" }, { "line": 1455, "key": "link_text", "value_expression": "title" }, { "line": 1455, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1450, "key": "package_kind", "value_expression": "'producer_listing'" }, { "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1452, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1451, "key": "is_odf_only", "value_expression": "False" }, { "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1441, "key": "producer_label", "value_expression": "\"Augustine's Virtual Organs\"" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "line_start": 1447, "line_end": 1455, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"producer_listing\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}]," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "line_start": 1449, "line_end": 1457, "text": " \"package_title\": title,\n \"package_kind\": \"producer_listing\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)},\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "line_start": 1448, "line_end": 1456, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"producer_listing\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "AugustineVirtualOrgansSamplesCrawler.parse_html", "line_start": 1438, "line_end": 1446, "text": " \"catalog\": {\"position\": position, \"section_label\": \"Sample sets\"},\n \"item\": {\n \"title\": title,\n \"producer_label\": \"Augustine's Virtual Organs\",\n \"description_excerpt\": f\"Public Augustine's Virtual Organs listing page for {title}.\",\n \"software_labels\": [\"Hauptwerk\"],\n \"engine_labels\": [\"Hauptwerk\"],\n \"access_model_hint\": \"free_or_low_cost\" if \"free\" in lower else \"producer_catalog\",\n \"availability_status_hint\": \"producer_listing\"," } ] } }, { "catalogue_id": "S004", "source_key": "bdo_tuning_systems", "name": "BDO Tuning Systems", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Hans Eugen Frischknecht; Jakob Schmid; hosted by Bund Deutscher Orgelbaumeister", "scope_description": "Historical and theoretical temperaments", "documentary_unit": "Tuning-system definition", "source_description": "The collection groups temperaments, offers sound examples and distinguishes a more precise calculation section. Its authors page credits Frischknecht with the conception, research and graphical presentation and Schmid with the website's implementation.", "source_description_references": [ "bdo_tuning_systems", "bdo_authors" ], "interpretation": "Temperament definitions support comparison of interval structures. They provide no evidence that a particular organ used a temperament.", "rights_observation": "No general licence was identified on the consulted index or authors page.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "bdo_tuning_systems", "bdo_authors" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler parses tuning definitions and can obtain separate commentary pages. Source parameter values and commentary have separate document identities.", "transformation": "The normalizer preserves a cent-value vector in a declared source tone order, group memberships and a precise-variant flag. Derived intervals are calculated in a separate routine. Commentary is partitioned into description, author note, source note and literature information. Source values and calculated musical quantities are therefore explicitly distinguishable.", "limitations": "A tuning definition does not demonstrate its use on an organ. A precise-variant suffix is an implementation convention that needs source interpretation. Calculated intervals should be cited as calculations with their parameters, not as verbatim source values." }, "implementation": { "class": "BdoTuningSystemsCrawler", "class_line": 29, "path": "aggregator/bdo_tuning_systems_crawler.py", "inheritance": [ "BdoTuningSystemsCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "tuning_id", "path": "aggregator/bdo_tuning_systems_crawler.py", "line": 37, "class": "BdoTuningSystemsCrawler" }, "record_label": { "value": "tuning systems", "path": "aggregator/bdo_tuning_systems_crawler.py", "line": 38, "class": "BdoTuningSystemsCrawler" } }, "methods": [ { "symbol": "BdoTuningSystemsCrawler._crawl_options", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 574, "line_end": 583, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_pages_from_options" ], "literal_get_calls": [ { "line": 582, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 582, "receiver": "source", "key": "max_items", "expression": "source.get('max_items')" } ], "html_queries": [], "dictionary_entries": [ { "line": 581, "key": "pages", "value_expression": "self._pages_from_options(source)" }, { "line": 582, "key": "limit", "value_expression": "source.get('limit') or source.get('max_items') or 0" } ], "subscript_assignments": [ { "line": 579, "value_line": 579, "key": "page", "target_expression": "source['page']", "value_expression": "str(target)" } ] }, { "symbol": "BdoTuningSystemsCrawler._fetch_html", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 134, "line_end": 138, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [], "literal_get_calls": [ { "line": 135, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler._fetch_optional_html", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 140, "line_end": 146, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [], "literal_get_calls": [ { "line": 141, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler._merge_record", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 383, "line_end": 394, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [], "literal_get_calls": [ { "line": 385, "receiver": "incoming", "key": "group_memberships", "expression": "incoming.get('group_memberships')" }, { "line": 389, "receiver": "incoming", "key": "group_key", "expression": "incoming.get('group_key')" }, { "line": 390, "receiver": "incoming", "key": "group_key", "expression": "incoming.get('group_key')" }, { "line": 391, "receiver": "incoming", "key": "group_title", "expression": "incoming.get('group_title')" }, { "line": 384, "receiver": "existing", "key": "group_memberships", "expression": "existing.get('group_memberships')" }, { "line": 389, "receiver": "existing", "key": "group_key", "expression": "existing.get('group_key')" }, { "line": 392, "receiver": "incoming", "key": "group_key", "expression": "incoming.get('group_key')" }, { "line": 393, "receiver": "incoming", "key": "group_key", "expression": "incoming.get('group_key')" }, { "line": 394, "receiver": "incoming", "key": "group_title", "expression": "incoming.get('group_title')" }, { "line": 392, "receiver": "existing", "key": "group_key", "expression": "existing.get('group_key')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 388, "value_line": 388, "key": "group_memberships", "target_expression": "existing['group_memberships']", "value_expression": "existing_groups" }, { "line": 390, "value_line": 390, "key": "group_key", "target_expression": "existing['group_key']", "value_expression": "incoming.get('group_key')" }, { "line": 391, "value_line": 391, "key": "group_title", "target_expression": "existing['group_title']", "value_expression": "incoming.get('group_title')" }, { "line": 393, "value_line": 393, "key": "group_key", "target_expression": "existing['group_key']", "value_expression": "incoming.get('group_key')" }, { "line": 394, "value_line": 394, "key": "group_title", "target_expression": "existing['group_title']", "value_expression": "incoming.get('group_title')" } ] }, { "symbol": "BdoTuningSystemsCrawler._record_from_href", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 326, "line_end": 358, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 329, "receiver": "params", "key": "label", "expression": "params.get('label')" }, { "line": 330, "receiver": "params", "key": "title", "expression": "params.get('title')" }, { "line": 331, "receiver": "params", "key": "centstr", "expression": "params.get('centstr')" } ], "html_queries": [], "dictionary_entries": [ { "line": 337, "key": "tuning_id", "value_expression": "label" }, { "line": 338, "key": "label", "value_expression": "label" }, { "line": 339, "key": "title", "value_expression": "title or label" }, { "line": 340, "key": "centstr", "value_expression": "centstr" }, { "line": 341, "key": "cent_values", "value_expression": "_parse_centstr(centstr)" }, { "line": 342, "key": "group_key", "value_expression": "group_key" }, { "line": 343, "key": "group_title", "value_expression": "group_title if group_key else ''" }, { "line": 344, "key": "group_memberships", "value_expression": "[{'group_key': group_key, 'group_title': group_title}] if group_key else []" }, { "line": 345, "key": "page_key", "value_expression": "page_key" }, { "line": 346, "key": "url", "value_expression": "source_url" }, { "line": 347, "key": "source_url", "value_expression": "source_url" }, { "line": 348, "key": "is_precise_variant", "value_expression": "label.endswith('_g')" }, { "line": 349, "key": "commentary_available", "value_expression": "0" }, { "line": 350, "key": "commentary_url", "value_expression": "''" }, { "line": 351, "key": "description", "value_expression": "''" }, { "line": 352, "key": "author_note", "value_expression": "''" }, { "line": 353, "key": "source_note", "value_expression": "''" }, { "line": 354, "key": "literature_text", "value_expression": "''" }, { "line": 355, "key": "raw_commentary_html", "value_expression": "''" }, { "line": 356, "key": "status", "value_expression": "'initiated'" }, { "line": 357, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 344, "key": "group_key", "value_expression": "group_key" }, { "line": 344, "key": "group_title", "value_expression": "group_title" } ], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler._record_from_link", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 315, "line_end": 324, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_clean_text", "_record_from_href" ], "literal_get_calls": [ { "line": 322, "receiver": "record", "key": "title", "expression": "record.get('title')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 320, "value_line": 320, "key": "raw_html", "target_expression": "record['raw_html']", "value_expression": "str(row or link)" }, { "line": 323, "value_line": 323, "key": "title", "target_expression": "record['title']", "value_expression": "linked_title" } ] }, { "symbol": "BdoTuningSystemsCrawler.crawl", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 87, "line_end": 132, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_attach_commentary_pages", "_crawl_options", "_fetch_html", "_merge_record", "_persist_cache_records", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "parse_catalog_page" ], "literal_get_calls": [ { "line": 89, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 91, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 118, "receiver": "self.config", "key": "fetch_commentary_pages", "expression": "self.config.get('fetch_commentary_pages')" }, { "line": 115, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler.normalized_batch_record", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 294, "line_end": 313, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 307, "value_line": 307, "key": "transform_run_id", "target_expression": "identity['transform_run_id']", "value_expression": "process_id" }, { "line": 308, "value_line": 308, "key": "artifact_ids", "target_expression": "identity['artifact_ids']", "value_expression": "list(artifact_ids or [])" }, { "line": 310, "value_line": 310, "key": "url", "target_expression": "record.setdefault('source_snapshot', {})['url']", "value_expression": "source_url" }, { "line": 312, "value_line": 312, "key": "retrieved_at", "target_expression": "record.setdefault('source', {})['retrieved_at']", "value_expression": "retrieved_at" } ] }, { "symbol": "BdoTuningSystemsCrawler.parse_catalog_page", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 148, "line_end": 159, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_page_context", "_record_from_link" ], "literal_get_calls": [ { "line": 153, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 152, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler.parse_commentary_page", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 179, "line_end": 213, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 188, "expression": "soup.find_all('dt')" } ], "dictionary_entries": [ { "line": 181, "key": "description", "value_expression": "[]" }, { "line": 181, "key": "author_note", "value_expression": "[]" }, { "line": 181, "key": "source_note", "value_expression": "[]" }, { "line": 181, "key": "literature", "value_expression": "[]" }, { "line": 183, "key": "beschreibung", "value_expression": "'description'" }, { "line": 184, "key": "autor", "value_expression": "'author_note'" }, { "line": 185, "key": "quelle", "value_expression": "'source_note'" }, { "line": 186, "key": "literatur", "value_expression": "'literature'" }, { "line": 203, "key": "available", "value_expression": "True" }, { "line": 204, "key": "url", "value_expression": "page_url" }, { "line": 205, "key": "label", "value_expression": "label" }, { "line": 206, "key": "description", "value_expression": "' '.join(sections['description'])" }, { "line": 207, "key": "author_note", "value_expression": "' '.join(sections['author_note'])" }, { "line": 208, "key": "source_note", "value_expression": "' '.join(sections['source_note'])" }, { "line": 209, "key": "literature_text", "value_expression": "'\\n'.join(literature_entries)" }, { "line": 210, "key": "literature_entries", "value_expression": "literature_entries" }, { "line": 211, "key": "raw_text", "value_expression": "self._clean_text(soup.get_text(' ', strip=True))" } ], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler.preflight_crawl", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 83, "line_end": 85, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_pages_from_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 85, "key": "ok", "value_expression": "bool(pages)" }, { "line": 85, "key": "pages", "value_expression": "pages" }, { "line": 85, "key": "page_count", "value_expression": "len(pages)" } ], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler.scrape_row_to_transfer_record", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 276, "line_end": 292, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [], "literal_get_calls": [ { "line": 285, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 289, "receiver": "raw_payload", "key": "raw_html", "expression": "raw_payload.get('raw_html')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler.test_label", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 260, "line_end": 274, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [ "_attach_commentary_pages", "_fetch_html", "parse_catalog_page", "transform" ], "literal_get_calls": [ { "line": 265, "receiver": "self.config", "key": "fetch_commentary_pages", "expression": "self.config.get('fetch_commentary_pages')" }, { "line": 261, "receiver": "self.config", "key": "index_page", "expression": "self.config.get('index_page', 'contents.htm')" }, { "line": 264, "receiver": "record", "key": "tuning_id", "expression": "record.get('tuning_id')" }, { "line": 264, "receiver": "record", "key": "label", "expression": "record.get('label')" } ], "html_queries": [], "dictionary_entries": [ { "line": 269, "key": "source", "value_expression": "self.source_key" }, { "line": 270, "key": "query", "value_expression": "{'label': wanted}" }, { "line": 271, "key": "parsed_count", "value_expression": "len(records)" }, { "line": 272, "key": "first_record", "value_expression": "sample" }, { "line": 273, "key": "first_transformed_record", "value_expression": "transformed" }, { "line": 270, "key": "label", "value_expression": "wanted" } ], "subscript_assignments": [] }, { "symbol": "BdoTuningSystemsCrawler.transform", "path": "aggregator/bdo_tuning_systems_crawler.py", "line_start": 161, "line_end": 177, "inherited": false, "file_sha256": "0671e570252fa9c8722bfc606add24bb5b7e41ddd44da821e4fad49be51a1b1f", "self_calls": [], "literal_get_calls": [ { "line": 163, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 175, "receiver": "payload", "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 176, "receiver": "payload", "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 174, "receiver": "payload", "key": "raw_html", "expression": "payload.get('raw_html')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "BdoTuningSystemsCrawler.transform", "path": "aggregator/bdo_tuning_systems_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 163, "kind": "input read" }, { "symbol": "BdoTuningSystemsCrawler.transform", "path": "aggregator/bdo_tuning_systems_crawler.py", "key": "crawl_run_id", "value_expression": "payload.get('crawl_run_id')", "line": 175, "kind": "input read" }, { "symbol": "BdoTuningSystemsCrawler.transform", "path": "aggregator/bdo_tuning_systems_crawler.py", "key": "transform_run_id", "value_expression": "payload.get('transform_run_id')", "line": 176, "kind": "input read" }, { "symbol": "BdoTuningSystemsCrawler.transform", "path": "aggregator/bdo_tuning_systems_crawler.py", "key": "raw_html", "value_expression": "payload.get('raw_html')", "line": 174, "kind": "input read" }, { "symbol": "BdoTuningSystemsCrawler.crawl", "path": "aggregator/bdo_tuning_systems_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 89, "kind": "input read" }, { "symbol": "BdoTuningSystemsCrawler.crawl", "path": "aggregator/bdo_tuning_systems_crawler.py", "key": "limit", "value_expression": "options.get('limit')", "line": 91, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/bdo_tuning_systems_crawler.py", "symbol": "BdoTuningSystemsCrawler.transform", "line_start": 161, "line_end": 168, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n payload = dict(raw_data or {})\n raw_json = payload.get(\"raw_data_json\")\n if isinstance(raw_json, str) and raw_json.strip():\n try:\n decoded = json.loads(raw_json)\n if isinstance(decoded, dict):\n decoded.update({key: value for key, value in payload.items() if value not in (None, \"\")})" }, { "path": "aggregator/bdo_tuning_systems_crawler.py", "symbol": "BdoTuningSystemsCrawler.transform", "line_start": 172, "line_end": 177, "text": " return normalize_tuning_system_record(\n payload,\n raw_content=payload.get(\"raw_html\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/bdo_tuning_systems_crawler.py", "symbol": "BdoTuningSystemsCrawler.transform", "line_start": 173, "line_end": 177, "text": " payload,\n raw_content=payload.get(\"raw_html\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/bdo_tuning_systems_crawler.py", "symbol": "BdoTuningSystemsCrawler.transform", "line_start": 171, "line_end": 177, "text": " pass\n return normalize_tuning_system_record(\n payload,\n raw_content=payload.get(\"raw_html\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/bdo_tuning_systems_crawler.py", "symbol": "BdoTuningSystemsCrawler.crawl", "line_start": 87, "line_end": 94, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n task_id = kwargs.get(\"task_id\")\n pages = options[\"pages\"]\n max_items = _safe_int(options.get(\"limit\"), default=0)\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting BDO tuning-system crawl.\", 0, max(1, len(pages)))\n\n records_by_id: Dict[str, Dict[str, Any]] = {}" }, { "path": "aggregator/bdo_tuning_systems_crawler.py", "symbol": "BdoTuningSystemsCrawler.crawl", "line_start": 88, "line_end": 96, "text": " options = self._crawl_options(target, kwargs)\n task_id = kwargs.get(\"task_id\")\n pages = options[\"pages\"]\n max_items = _safe_int(options.get(\"limit\"), default=0)\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting BDO tuning-system crawl.\", 0, max(1, len(pages)))\n\n records_by_id: Dict[str, Dict[str, Any]] = {}\n try:\n for index, page in enumerate(pages, start=1):" } ] } }, { "catalogue_id": "S005", "source_key": "binauralpipes_instruments", "name": "Binaural Pipes Instruments", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Dietrich Wierczeyko / Binauralpipes", "scope_description": "Selected organs represented in a sampling project", "documentary_unit": "Instrument or prospective sample-set listing", "source_description": "The instruments page lists places, builders, dates and compact stop/manual counts. It includes a section for sample sets in preparation and credits Dietrich Wierczeyko in its copyright notice.", "source_description_references": [ "binauralpipes_instruments" ], "interpretation": "A listed organ is not necessarily an available download. Preparation, release and subsequent revision should be distinct states. The project's selection cannot measure the distribution of surviving organs.", "rights_observation": "Copyright is asserted; the consulted list does not establish download or redistribution terms.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "binauralpipes_instruments" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects article content and links to instrument pages on the producer host. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is instrument/product listings. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A listed instrument may describe a planned product. Availability and release state must come from the individual item. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "BinauralPipesInstrumentsCrawler", "class_line": 1148, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "BinauralPipesInstrumentsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "binauralpipes_instruments", "path": "aggregator/grandorgue_library_crawler.py", "line": 1149, "class": "BinauralPipesInstrumentsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 1151, "class": "BinauralPipesInstrumentsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1153, "line_end": 1208, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1159, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 1155, "expression": "soup.select_one('article')" }, { "line": 1158, "expression": "article.select('a[href*=\"binauralpipes.com/\"]')" } ], "dictionary_entries": [ { "line": 1174, "key": "catalog", "value_expression": "{'position': position}" }, { "line": 1175, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Binaural Pipes', 'description_excerpt': _excerpt(caption), 'access_model_hint': 'free_download_or_donation_supported', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1182, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1188, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': title, 'builder_label': builder, 'year_label': year, 'manual_count': manual_count, 'pedal_present': '/P' in caption, 'stop_count': stop_count}" }, { "line": 1197, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1198, "key": "extensions", "value_expression": "{'source_fragment_html': str(container)}" }, { "line": 1174, "key": "position", "value_expression": "position" }, { "line": 1176, "key": "title", "value_expression": "title" }, { "line": 1177, "key": "producer_label", "value_expression": "'Binaural Pipes'" }, { "line": 1178, "key": "description_excerpt", "value_expression": "_excerpt(caption)" }, { "line": 1179, "key": "access_model_hint", "value_expression": "'free_download_or_donation_supported'" }, { "line": 1180, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1183, "key": "package_title", "value_expression": "title" }, { "line": 1184, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1185, "key": "is_odf_only", "value_expression": "False" }, { "line": 1186, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1189, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 1190, "key": "represented_organ_label", "value_expression": "title" }, { "line": 1191, "key": "builder_label", "value_expression": "builder" }, { "line": 1192, "key": "year_label", "value_expression": "year" }, { "line": 1193, "key": "manual_count", "value_expression": "manual_count" }, { "line": 1194, "key": "pedal_present", "value_expression": "'/P' in caption" }, { "line": 1195, "key": "stop_count", "value_expression": "stop_count" }, { "line": 1198, "key": "source_fragment_html", "value_expression": "str(container)" }, { "line": 1197, "key": "url", "value_expression": "detail_url" }, { "line": 1197, "key": "link_text", "value_expression": "title" }, { "line": 1197, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1195, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1193, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1184, "key": "package_kind", "value_expression": "'sample_set'" }, { "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1186, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1185, "key": "is_odf_only", "value_expression": "False" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "line_start": 1192, "line_end": 1200, "text": " \"year_label\": year,\n \"manual_count\": manual_count,\n \"pedal_present\": \"/P\" in caption,\n \"stop_count\": stop_count,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(container)},\n },\n primary_id=slugify(path)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "line_start": 1190, "line_end": 1198, "text": " \"represented_organ_label\": title,\n \"builder_label\": builder,\n \"year_label\": year,\n \"manual_count\": manual_count,\n \"pedal_present\": \"/P\" in caption,\n \"stop_count\": stop_count,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(container)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "line_start": 1181, "line_end": 1189, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\"," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "line_start": 1183, "line_end": 1191, "text": " \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title,\n \"builder_label\": builder," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "BinauralPipesInstrumentsCrawler.parse_html", "line_start": 1182, "line_end": 1190, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title," } ] } }, { "catalogue_id": "S006", "source_key": "contrebombarde_organs", "name": "Contrebombarde Sample Set Library", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Contrebombarde Concert Hall", "scope_description": "Cross-producer virtual organ catalogue associated with a recording community", "documentary_unit": "Sample-set registry entry", "source_description": "The search interface distinguishes sample-set producer from physical organ builder and offers filters for style, manuals and construction period. The interface therefore distinguishes the virtual product's producer from the instrument builder.", "source_description_references": [ "contrebombarde_organs" ], "interpretation": "Use the registry for product discovery and links to performances. Availability, price and licensing should be cited to the producer's corresponding release. A recording associated with a set does not document a performance on the physical organ.", "rights_observation": "Registry access was public; no common licence for the linked products or performances was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "contrebombarde_organs" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects organ detail/view links and organ-card, article or table containers. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is registry items reached from search and detail pages. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Registry descriptions and producer claims have different authorship; search traversal does not validate product availability. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "ContrebombardeOrgansCrawler", "class_line": 649, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "ContrebombardeOrgansCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "contrebombarde_organs", "path": "aggregator/grandorgue_library_crawler.py", "line": 650, "class": "ContrebombardeOrgansCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 652, "class": "ContrebombardeOrgansCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" }, "search_url": { "value": "https://www.contrebombarde.com/concerthall/organs/search", "path": "aggregator/grandorgue_library_crawler.py", "line": 653, "class": "ContrebombardeOrgansCrawler" }, "search_results_url": { "value": "https://www.contrebombarde.com/concerthall/organs/search_results", "path": "aggregator/grandorgue_library_crawler.py", "line": 654, "class": "ContrebombardeOrgansCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "ContrebombardeOrgansCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 657, "line_end": 712, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 659, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 660, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 658, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 661, "receiver": "kwargs", "key": "form_data", "expression": "kwargs.get('form_data')" }, { "line": 661, "receiver": "self.config", "key": "form_data", "expression": "self.config.get('form_data')" }, { "line": 662, "receiver": "kwargs", "key": "parse_url", "expression": "kwargs.get('parse_url')" }, { "line": 662, "receiver": "self.config", "key": "search_url", "expression": "self.config.get('search_url')" }, { "line": 665, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 667, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 697, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 698, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 664, "receiver": "kwargs", "key": "method", "expression": "kwargs.get('method')" }, { "line": 664, "receiver": "self.config", "key": "method", "expression": "self.config.get('method')" }, { "line": 700, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 697, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 698, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 700, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 661, "key": "software", "value_expression": "self.grandorgue_software_id" }, { "line": 661, "key": "name", "value_expression": "''" } ], "subscript_assignments": [] }, { "symbol": "ContrebombardeOrgansCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 714, "line_end": 785, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 743, "receiver": "fields", "key": "website", "expression": "fields.get('website')" }, { "line": 743, "receiver": "fields", "key": "producer website", "expression": "fields.get('producer website')" }, { "line": 733, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" }, { "line": 733, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" }, { "line": 734, "receiver": "container", "key": "data-organ-id", "expression": "container.get('data-organ-id')" }, { "line": 737, "receiver": "fields", "key": "software", "expression": "fields.get('software')" }, { "line": 757, "receiver": "fields", "key": "producer", "expression": "fields.get('producer')" }, { "line": 767, "receiver": "fields", "key": "builder", "expression": "fields.get('builder')" }, { "line": 768, "receiver": "fields", "key": "country", "expression": "fields.get('country')" }, { "line": 772, "receiver": "fields", "key": "style", "expression": "fields.get('style')" }, { "line": 769, "receiver": "fields", "key": "manuals", "expression": "fields.get('manuals')" }, { "line": 770, "receiver": "fields", "key": "pedal", "expression": "fields.get('pedal')" }, { "line": 771, "receiver": "fields", "key": "stops", "expression": "fields.get('stops')" }, { "line": 765, "receiver": "fields", "key": "builder", "expression": "fields.get('builder')" }, { "line": 765, "receiver": "fields", "key": "country", "expression": "fields.get('country')" } ], "html_queries": [ { "line": 717, "expression": "soup.select('a[href*=\"/organs/view_organ/\"], a[href*=\"/organs/details/\"]')" }, { "line": 721, "expression": "soup.select('[data-organ-id], .organ, .organ-card, article, tr')" }, { "line": 722, "expression": "soup.select_one('h1')" }, { "line": 727, "expression": "node.select_one('a[href*=\"/organs/\"], h1 a, h2 a, h3 a')" } ], "dictionary_entries": [ { "line": 745, "key": "url", "value_expression": "website" }, { "line": 745, "key": "link_text", "value_expression": "'Website'" }, { "line": 745, "key": "is_product_homepage_candidate", "value_expression": "True" }, { "line": 749, "key": "catalog", "value_expression": "{'position': position, 'query_url': self.search_url, 'result_url': source_url, 'facet_filters': {'software': 'GrandOrgue', 'software_id': self.grandorgue_software_id}}" }, { "line": 755, "key": "item", "value_expression": "{'title': title, 'producer_label': fields.get('producer'), 'description_excerpt': _excerpt(text), 'software_labels': software, 'engine_labels': software, 'availability_status_hint': 'catalog_only'}" }, { "line": 763, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'catalog_entry'}" }, { "line": 764, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ' if fields.get('builder') or fields.get('country') else 'generic_pipe_organ', 'represented_organ_label': title, 'builder_label': fields.get('builder'), 'country_label': fields.get('country'), 'manual_count': _first_int(fields.get('manuals')), 'pedal_present': _bool_from_text(fields.get('pedal')), 'stop_count': _first_int(fields.get('stops')), 'style_label': fields.get('style')}" }, { "line": 774, "key": "links", "value_expression": "links" }, { "line": 775, "key": "extensions", "value_expression": "{'source_fragment_html': str(container)}" }, { "line": 750, "key": "position", "value_expression": "position" }, { "line": 751, "key": "query_url", "value_expression": "self.search_url" }, { "line": 752, "key": "result_url", "value_expression": "source_url" }, { "line": 753, "key": "facet_filters", "value_expression": "{'software': 'GrandOrgue', 'software_id': self.grandorgue_software_id}" }, { "line": 756, "key": "title", "value_expression": "title" }, { "line": 757, "key": "producer_label", "value_expression": "fields.get('producer')" }, { "line": 758, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 759, "key": "software_labels", "value_expression": "software" }, { "line": 760, "key": "engine_labels", "value_expression": "software" }, { "line": 761, "key": "availability_status_hint", "value_expression": "'catalog_only'" }, { "line": 763, "key": "package_title", "value_expression": "title" }, { "line": 763, "key": "package_kind", "value_expression": "'catalog_entry'" }, { "line": 765, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ' if fields.get('builder') or fields.get('country') else 'generic_pipe_organ'" }, { "line": 766, "key": "represented_organ_label", "value_expression": "title" }, { "line": 767, "key": "builder_label", "value_expression": "fields.get('builder')" }, { "line": 768, "key": "country_label", "value_expression": "fields.get('country')" }, { "line": 769, "key": "manual_count", "value_expression": "_first_int(fields.get('manuals'))" }, { "line": 770, "key": "pedal_present", "value_expression": "_bool_from_text(fields.get('pedal'))" }, { "line": 771, "key": "stop_count", "value_expression": "_first_int(fields.get('stops'))" }, { "line": 772, "key": "style_label", "value_expression": "fields.get('style')" }, { "line": 775, "key": "source_fragment_html", "value_expression": "str(container)" }, { "line": 753, "key": "software", "value_expression": "'GrandOrgue'" }, { "line": 753, "key": "software_id", "value_expression": "self.grandorgue_software_id" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "ContrebombardeOrgansCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 771, "key": "stop_count", "value_expression": "_first_int(fields.get('stops'))" }, { "symbol": "ContrebombardeOrgansCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 769, "key": "manual_count", "value_expression": "_first_int(fields.get('manuals'))" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "ContrebombardeOrgansCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 763, "key": "package_kind", "value_expression": "'catalog_entry'" }, { "symbol": "ContrebombardeOrgansCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 757, "key": "producer_label", "value_expression": "fields.get('producer')" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "ContrebombardeOrgansCrawler.parse_html", "line_start": 768, "line_end": 776, "text": " \"country_label\": fields.get(\"country\"),\n \"manual_count\": _first_int(fields.get(\"manuals\")),\n \"pedal_present\": _bool_from_text(fields.get(\"pedal\")),\n \"stop_count\": _first_int(fields.get(\"stops\")),\n \"style_label\": fields.get(\"style\"),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(container)},\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "ContrebombardeOrgansCrawler.parse_html", "line_start": 766, "line_end": 774, "text": " \"represented_organ_label\": title,\n \"builder_label\": fields.get(\"builder\"),\n \"country_label\": fields.get(\"country\"),\n \"manual_count\": _first_int(fields.get(\"manuals\")),\n \"pedal_present\": _bool_from_text(fields.get(\"pedal\")),\n \"stop_count\": _first_int(fields.get(\"stops\")),\n \"style_label\": fields.get(\"style\"),\n },\n \"links\": links," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "ContrebombardeOrgansCrawler.parse_html", "line_start": 760, "line_end": 768, "text": " \"engine_labels\": software,\n \"availability_status_hint\": \"catalog_only\",\n },\n \"package\": {\"package_title\": title, \"package_kind\": \"catalog_entry\"},\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\" if fields.get(\"builder\") or fields.get(\"country\") else \"generic_pipe_organ\",\n \"represented_organ_label\": title,\n \"builder_label\": fields.get(\"builder\"),\n \"country_label\": fields.get(\"country\")," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "ContrebombardeOrgansCrawler.parse_html", "line_start": 754, "line_end": 762, "text": " },\n \"item\": {\n \"title\": title,\n \"producer_label\": fields.get(\"producer\"),\n \"description_excerpt\": _excerpt(text),\n \"software_labels\": software,\n \"engine_labels\": software,\n \"availability_status_hint\": \"catalog_only\",\n }," } ] } }, { "catalogue_id": "S007", "source_key": "digital_sheet_music", "name": "Digital Sheet Music Sources", "selection_role": "registered_source", "parent_source_key": null, "attribution": "MODAVIS Aggregator (composite connector)", "scope_description": "PDMX; Little Organ Book; IMSLP/WIMA; Mutopia; direct MuseScore records", "documentary_unit": "Collection-qualified score record and file assets", "source_description": "This key denotes a connector spanning five named collections. Its configured MusicXML.com address is a directory of music resources, not the publisher of all the scores. The five collection definitions are recorded separately in this package.", "source_description_references": [ "digital_sheet_music", "repo_score_collections" ], "interpretation": "Cite the score's collection, edition and file version. Neither a composition title nor the connector key identifies an edition.", "rights_observation": "Rights differ by collection, edition and file. The MusicXML directory supplies no umbrella permission.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "digital_sheet_music", "repo_score_collections" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "A single registered connector dispatches to five named collections. Dataset imports, web catalogues and direct score routes have different acquisition conditions; the MusicXML directory is not their common publisher.", "transformation": "The score normalizer separates work metadata from downloadable assets, formats, collection identity and licence statements. It preserves collection-specific data. In the PDMX path it suppresses a numeric instrumentation value when that value simply repeats the track count. This prevents a numeric count being relabelled as instrument names.", "limitations": "Work, arrangement, edition, encoding and downloadable file are different units. A format label does not guarantee that a file was decoded or compared musically. Collection-specific licences cannot be replaced by an umbrella connector licence." }, "implementation": { "class": "DigitalSheetMusicCrawler", "class_line": 51, "path": "aggregator/digital_sheet_music_crawler.py", "inheritance": [ "DigitalSheetMusicCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "score_id", "path": "aggregator/digital_sheet_music_crawler.py", "line": 58, "class": "DigitalSheetMusicCrawler" }, "record_label": { "value": "digital scores", "path": "aggregator/digital_sheet_music_crawler.py", "line": 59, "class": "DigitalSheetMusicCrawler" } }, "methods": [ { "symbol": "DigitalSheetMusicCrawler._assets_from_html", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 857, "line_end": 905, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 865, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 864, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 887, "key": "url", "value_expression": "href" }, { "line": 887, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 887, "key": "format", "value_expression": "info['format']" }, { "line": 887, "key": "label", "value_expression": "text or info['format']" }, { "line": 897, "key": "url", "value_expression": "href" }, { "line": 898, "key": "alternate_urls", "value_expression": "_imslp_mirror_candidates(href, imslp_file_id)" }, { "line": 899, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 900, "key": "format", "value_expression": "info['format']" }, { "line": 901, "key": "label", "value_expression": "text or info['format']" }, { "line": 902, "key": "source_file_id", "value_expression": "imslp_file_id" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_collection", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 318, "line_end": 329, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_imslp_wima", "_crawl_musescore_direct", "_crawl_mutopia", "_crawl_orgelbuechlein", "_crawl_pdmx" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_imslp_wima", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 457, "line_end": 484, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 459, "receiver": "options", "key": "imslp_seed_urls", "expression": "options.get('imslp_seed_urls')" }, { "line": 460, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 461, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 479, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [], "dictionary_entries": [ { "line": 480, "key": "html_excerpt", "value_expression": "html[:10000]" }, { "line": 480, "key": "seed_url", "value_expression": "url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_musescore_direct", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 513, "line_end": 563, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 514, "receiver": "options", "key": "musescore_direct_urls", "expression": "options.get('musescore_direct_urls')" }, { "line": 515, "receiver": "self.config", "key": "musescore_credentials_env", "expression": "self.config.get('musescore_credentials_env')" }, { "line": 539, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 543, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 532, "key": "credential_required", "value_expression": "True" }, { "line": 533, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "credential_required", "value_expression": "True" }, { "line": 559, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "http_status", "value_expression": "response.status_code" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_mutopia", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 486, "line_end": 511, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_mutopia_next_url", "_parse_mutopia_listing", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 487, "receiver": "self.config", "key": "mutopia_collection_url", "expression": "self.config.get('mutopia_collection_url')" }, { "line": 488, "receiver": "self.config", "key": "mutopia_max_pages", "expression": "self.config.get('mutopia_max_pages', 0)" }, { "line": 496, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 500, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_options", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 280, "line_end": 316, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "source", "key": "collection", "expression": "source.get('collection')" }, { "line": 286, "receiver": "source", "key": "collections", "expression": "source.get('collections')" }, { "line": 293, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 295, "receiver": "source", "key": "organ_only", "expression": "source.get('organ_only')" }, { "line": 302, "receiver": "source", "key": "pdmx_enable_archive_streaming", "expression": "source.get('pdmx_enable_archive_streaming')" }, { "line": 288, "receiver": "self.config", "key": "default_collections", "expression": "self.config.get('default_collections')" }, { "line": 297, "receiver": "source", "key": "pdmx_csv_path", "expression": "source.get('pdmx_csv_path')" }, { "line": 297, "receiver": "self.config", "key": "pdmx_csv_path", "expression": "self.config.get('pdmx_csv_path')" }, { "line": 299, "receiver": "source", "key": "pdmx_root_dir", "expression": "source.get('pdmx_root_dir')" }, { "line": 299, "receiver": "self.config", "key": "pdmx_root_dir", "expression": "self.config.get('pdmx_root_dir')" }, { "line": 300, "receiver": "source", "key": "pdmx_zenodo_record_api_url", "expression": "source.get('pdmx_zenodo_record_api_url')" }, { "line": 300, "receiver": "self.config", "key": "pdmx_zenodo_record_api_url", "expression": "self.config.get('pdmx_zenodo_record_api_url')" }, { "line": 303, "receiver": "self.config", "key": "pdmx_enable_archive_streaming", "expression": "self.config.get('pdmx_enable_archive_streaming', False)" }, { "line": 306, "receiver": "source", "key": "pdmx_archive_stream_max_mb", "expression": "source.get('pdmx_archive_stream_max_mb')" }, { "line": 306, "receiver": "self.config", "key": "pdmx_archive_stream_max_mb", "expression": "self.config.get('pdmx_archive_stream_max_mb')" }, { "line": 314, "receiver": "source", "key": "imslp_seed_urls", "expression": "source.get('imslp_seed_urls')" }, { "line": 315, "receiver": "source", "key": "musescore_direct_urls", "expression": "source.get('musescore_direct_urls')" }, { "line": 310, "receiver": "source", "key": "pdmx_mxl_archive_url", "expression": "source.get('pdmx_mxl_archive_url')" }, { "line": 310, "receiver": "self.config", "key": "pdmx_mxl_archive_url", "expression": "self.config.get('pdmx_mxl_archive_url')" }, { "line": 311, "receiver": "source", "key": "pdmx_mid_archive_url", "expression": "source.get('pdmx_mid_archive_url')" }, { "line": 311, "receiver": "self.config", "key": "pdmx_mid_archive_url", "expression": "self.config.get('pdmx_mid_archive_url')" }, { "line": 312, "receiver": "source", "key": "pdmx_pdf_archive_url", "expression": "source.get('pdmx_pdf_archive_url')" }, { "line": 312, "receiver": "self.config", "key": "pdmx_pdf_archive_url", "expression": "self.config.get('pdmx_pdf_archive_url')" }, { "line": 314, "receiver": "self.config", "key": "imslp_seed_urls", "expression": "self.config.get('imslp_seed_urls')" }, { "line": 315, "receiver": "self.config", "key": "musescore_direct_urls", "expression": "self.config.get('musescore_direct_urls')" }, { "line": 294, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 296, "receiver": "source", "key": "license_filter", "expression": "source.get('license_filter')" }, { "line": 298, "receiver": "source", "key": "pdmx_mode", "expression": "source.get('pdmx_mode')" }, { "line": 298, "receiver": "self.config", "key": "pdmx_mode", "expression": "self.config.get('pdmx_mode')" } ], "html_queries": [], "dictionary_entries": [ { "line": 292, "key": "collections", "value_expression": "[collection for collection in collections if collection in SOURCE_COLLECTIONS]" }, { "line": 293, "key": "limit", "value_expression": "_safe_int(source.get('limit'), default=0)" }, { "line": 294, "key": "query", "value_expression": "str(source.get('query') or '').strip().lower()" }, { "line": 295, "key": "organ_only", "value_expression": "_read_bool(source.get('organ_only'), True)" }, { "line": 296, "key": "license_filter", "value_expression": "str(source.get('license_filter') or '').strip().lower()" }, { "line": 297, "key": "pdmx_csv_path", "value_expression": "str(source.get('pdmx_csv_path') or self.config.get('pdmx_csv_path') or '')" }, { "line": 298, "key": "pdmx_mode", "value_expression": "str(source.get('pdmx_mode') or self.config.get('pdmx_mode') or 'rows').strip().lower()" }, { "line": 299, "key": "pdmx_root_dir", "value_expression": "str(source.get('pdmx_root_dir') or self.config.get('pdmx_root_dir') or '')" }, { "line": 300, "key": "pdmx_zenodo_record_api_url", "value_expression": "str(source.get('pdmx_zenodo_record_api_url') or self.config.get('pdmx_zenodo_record_api_url') or '')" }, { "line": 301, "key": "pdmx_enable_archive_streaming", "value_expression": "_read_bool(source.get('pdmx_enable_archive_streaming'), bool(self.config.get('pdmx_enable_archive_streaming', False)))" }, { "line": 305, "key": "pdmx_archive_stream_max_mb", "value_expression": "_safe_int(source.get('pdmx_archive_stream_max_mb') or self.config.get('pdmx_archive_stream_max_mb'), default=256)" }, { "line": 309, "key": "pdmx_archive_urls", "value_expression": "{'mxl': str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or ''), 'mid': str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or ''), 'pdf': str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')}" }, { "line": 314, "key": "imslp_seed_urls", "value_expression": "_split_lines(source.get('imslp_seed_urls')) or list(self.config.get('imslp_seed_urls') or [])" }, { "line": 315, "key": "musescore_direct_urls", "value_expression": "_split_lines(source.get('musescore_direct_urls')) or list(self.config.get('musescore_direct_urls') or [])" }, { "line": 310, "key": "mxl", "value_expression": "str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or '')" }, { "line": 311, "key": "mid", "value_expression": "str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or '')" }, { "line": 312, "key": "pdf", "value_expression": "str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')" } ], "subscript_assignments": [ { "line": 285, "value_line": 285, "key": "collection", "target_expression": "source['collection']", "value_expression": "str(target)" } ] }, { "symbol": "DigitalSheetMusicCrawler._crawl_orgelbuechlein", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 416, "line_end": 455, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 417, "receiver": "self.config", "key": "orgelbuechlein_api_url", "expression": "self.config.get('orgelbuechlein_api_url')" }, { "line": 429, "receiver": "item", "key": "download_url", "expression": "item.get('download_url')" }, { "line": 429, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 418, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 423, "receiver": "item", "key": "name", "expression": "item.get('name')" }, { "line": 436, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 453, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "github_item", "value_expression": "item" }, { "line": 442, "key": "url", "value_expression": "raw_url" }, { "line": 442, "key": "file_name", "value_expression": "name" }, { "line": 442, "key": "format", "value_expression": "asset_format_from_url(name)['format']" }, { "line": 442, "key": "label", "value_expression": "'MusicXML'" }, { "line": 444, "key": "url", "value_expression": "raw_url.replace('/little-organ-book-mxl/', '/little-organ-book-pdf/').rsplit('.', 1)[0] + '.pdf'" }, { "line": 445, "key": "file_name", "value_expression": "f'{stem}.pdf'" }, { "line": 446, "key": "format", "value_expression": "'pdf'" }, { "line": 447, "key": "label", "value_expression": "'PDF'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 331, "line_end": 364, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_pdmx_zenodo_inventory", "_pdmx_assets", "_pdmx_row_matches", "_raw_record", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 332, "receiver": "options", "key": "pdmx_mode", "expression": "options.get('pdmx_mode')" }, { "line": 338, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 334, "receiver": "options", "key": "pdmx_csv_path", "expression": "options.get('pdmx_csv_path')" }, { "line": 337, "receiver": "options", "key": "pdmx_root_dir", "expression": "options.get('pdmx_root_dir')" }, { "line": 338, "receiver": "self.config", "key": "pdmx_max_rows", "expression": "self.config.get('pdmx_max_rows')" }, { "line": 348, "receiver": "row", "key": "path", "expression": "row.get('path')" }, { "line": 348, "receiver": "row", "key": "mxl", "expression": "row.get('mxl')" }, { "line": 348, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "song_name", "expression": "row.get('song_name')" }, { "line": 354, "receiver": "row", "key": "composer_name", "expression": "row.get('composer_name')" }, { "line": 354, "receiver": "row", "key": "artist_name", "expression": "row.get('artist_name')" }, { "line": 356, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 357, "receiver": "row", "key": "license_url", "expression": "row.get('license_url')" }, { "line": 359, "receiver": "row", "key": "instrumentation", "expression": "row.get('instrumentation')" }, { "line": 359, "receiver": "row", "key": "instruments", "expression": "row.get('instruments')" }, { "line": 359, "receiver": "row", "key": "instrument", "expression": "row.get('instrument')" } ], "html_queries": [], "dictionary_entries": [ { "line": 361, "key": "pdmx", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx_zenodo_inventory", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 366, "line_end": 414, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 373, "receiver": "payload", "key": "files", "expression": "payload.get('files')" }, { "line": 390, "receiver": "payload.get('metadata', {})", "key": "title", "expression": "payload.get('metadata', {}).get('title')" }, { "line": 367, "receiver": "options", "key": "pdmx_zenodo_record_api_url", "expression": "options.get('pdmx_zenodo_record_api_url')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "content", "expression": "item.get('links', {}).get('content')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "self", "expression": "item.get('links', {}).get('self')" }, { "line": 370, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 376, "receiver": "item", "key": "key", "expression": "item.get('key')" }, { "line": 386, "receiver": "item", "key": "size", "expression": "item.get('size')" }, { "line": 390, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 397, "receiver": "payload.get('links', {})", "key": "html", "expression": "payload.get('links', {}).get('html')" }, { "line": 398, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "id", "expression": "payload.get('metadata', {}).get('license', {}).get('id')" }, { "line": 399, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "url", "expression": "payload.get('metadata', {}).get('license', {}).get('url')" }, { "line": 406, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 407, "receiver": "payload", "key": "doi", "expression": "payload.get('doi')" }, { "line": 393, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 397, "receiver": "payload", "key": "links", "expression": "payload.get('links', {})" }, { "line": 398, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 399, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 398, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 399, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 382, "key": "url", "value_expression": "content_url" }, { "line": 383, "key": "file_name", "value_expression": "key" }, { "line": 384, "key": "format", "value_expression": "asset_format_from_url(key)['format']" }, { "line": 385, "key": "label", "value_expression": "'Zenodo dataset file'" }, { "line": 386, "key": "content_size_bytes", "value_expression": "item.get('size')" }, { "line": 387, "key": "download_status", "value_expression": "'dataset_snapshot_required'" }, { "line": 405, "key": "zenodo_record", "value_expression": "{'id': payload.get('id'), 'doi': payload.get('doi'), 'api_url': api_url, 'files': files}" }, { "line": 411, "key": "access_note", "value_expression": "'PDMX requires an explicit local snapshot path for score-level crawling to avoid automatic multi-GB dataset downloads.'" }, { "line": 406, "key": "id", "value_expression": "payload.get('id')" }, { "line": 407, "key": "doi", "value_expression": "payload.get('doi')" }, { "line": 408, "key": "api_url", "value_expression": "api_url" }, { "line": 409, "key": "files", "value_expression": "files" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._extract_tar_gz_member_from_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 780, "line_end": 812, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 785, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._mutopia_next_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 990, "line_end": 995, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 994, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 992, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._parse_mutopia_listing", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 907, "line_end": 988, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 939, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 951, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 912, "receiver": "row", "key": "class", "expression": "row.get('class')" }, { "line": 980, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [ { "line": 911, "expression": "soup.select('table.result-table')" }, { "line": 915, "expression": "rows[0].find_all('td', recursive=False)" }, { "line": 937, "expression": "table.find_all('a', href=True)" }, { "line": 947, "expression": "table.find_all('a', href=True)" }, { "line": 928, "expression": "rows[1].find_all('td', recursive=False)" }, { "line": 912, "expression": "table.find_all('tr', recursive=False)" } ], "dictionary_entries": [ { "line": 972, "key": "source_collection", "value_expression": "'mutopia'" }, { "line": 973, "key": "title", "value_expression": "title" }, { "line": 974, "key": "composer", "value_expression": "composer" }, { "line": 975, "key": "source_url", "value_expression": "piece_url or identity_url" }, { "line": 976, "key": "license_name", "value_expression": "license_name" }, { "line": 977, "key": "license_url", "value_expression": "license_url" }, { "line": 978, "key": "catalog_number", "value_expression": "catalog_number or _extract_bwv(title)" }, { "line": 979, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 980, "key": "assets", "value_expression": "_limited(assets, int(self.config.get('max_assets_per_record', 0) or 0))" }, { "line": 981, "key": "raw_data", "value_expression": "{'mutopia_listing_title': title, 'mutopia_piece_url': piece_url, 'mutopia_listing_url': source_url}" }, { "line": 954, "key": "url", "value_expression": "href" }, { "line": 955, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 956, "key": "format", "value_expression": "asset_format_from_url(href)['format']" }, { "line": 957, "key": "label", "value_expression": "label" }, { "line": 982, "key": "mutopia_listing_title", "value_expression": "title" }, { "line": 983, "key": "mutopia_piece_url", "value_expression": "piece_url" }, { "line": 984, "key": "mutopia_listing_url", "value_expression": "source_url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_assets", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 832, "line_end": 855, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 835, "receiver": "options", "key": "pdmx_enable_archive_streaming", "expression": "options.get('pdmx_enable_archive_streaming')" }, { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 850, "receiver": "options", "key": "pdmx_archive_stream_max_mb", "expression": "options.get('pdmx_archive_stream_max_mb')" } ], "html_queries": [], "dictionary_entries": [ { "line": 841, "key": "file_name", "value_expression": "Path(value).name" }, { "line": 841, "key": "format", "value_expression": "asset_format_from_url(value)['format']" }, { "line": 841, "key": "label", "value_expression": "label" } ], "subscript_assignments": [ { "line": 843, "value_line": 843, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 845, "value_line": 845, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 848, "value_line": 848, "key": "archive_url", "target_expression": "asset['archive_url']", "value_expression": "archive_url" }, { "line": 849, "value_line": 849, "key": "archive_member_path", "target_expression": "asset['archive_member_path']", "value_expression": "value.replace('./', '', 1)" }, { "line": 850, "value_line": 850, "key": "archive_stream_max_mb", "target_expression": "asset['archive_stream_max_mb']", "value_expression": "options.get('pdmx_archive_stream_max_mb')" }, { "line": 851, "value_line": 851, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'archive_stream_required'" }, { "line": 853, "value_line": 853, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'local_snapshot_missing'" } ] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_row_matches", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 814, "line_end": 830, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 821, "receiver": "options", "key": "license_filter", "expression": "options.get('license_filter')" }, { "line": 824, "receiver": "options", "key": "query", "expression": "options.get('query')" }, { "line": 826, "receiver": "options", "key": "organ_only", "expression": "options.get('organ_only')" }, { "line": 827, "receiver": "self.config", "key": "pdmx_include_track_19", "expression": "self.config.get('pdmx_include_track_19', True)" }, { "line": 815, "receiver": "row", "key": "subset:deduplicated", "expression": "row.get('subset:deduplicated', '')" }, { "line": 817, "receiver": "row", "key": "subset:no_license_conflict", "expression": "row.get('subset:no_license_conflict', '')" }, { "line": 819, "receiver": "row", "key": "subset:all_valid", "expression": "row.get('subset:all_valid', '')" }, { "line": 821, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 827, "receiver": "row", "key": "tracks", "expression": "row.get('tracks')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._raw_record", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 565, "line_end": 599, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 581, "receiver": "asset", "key": "format", "expression": "asset.get('format')" }, { "line": 581, "receiver": "asset", "key": "url", "expression": "asset.get('url')" }, { "line": 581, "receiver": "asset", "key": "local_path", "expression": "asset.get('local_path')" } ], "html_queries": [], "dictionary_entries": [ { "line": 583, "key": "score_id", "value_expression": "score_id" }, { "line": 584, "key": "source_collection", "value_expression": "source_collection" }, { "line": 585, "key": "title", "value_expression": "title" }, { "line": 586, "key": "composer", "value_expression": "composer" }, { "line": 587, "key": "source_url", "value_expression": "source_url" }, { "line": 588, "key": "url", "value_expression": "source_url" }, { "line": 589, "key": "license_name", "value_expression": "license_name" }, { "line": 590, "key": "license_url", "value_expression": "license_url" }, { "line": 591, "key": "catalog_number", "value_expression": "catalog_number" }, { "line": 592, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 593, "key": "formats", "value_expression": "','.join(formats)" }, { "line": 594, "key": "status", "value_expression": "status" }, { "line": 595, "key": "assets", "value_expression": "assets" }, { "line": 596, "key": "raw_data_json", "value_expression": "json.dumps({**raw_data, 'assets': assets}, ensure_ascii=False)" }, { "line": 597, "key": "raw_data", "value_expression": "raw_data" }, { "line": 598, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 596, "key": "assets", "value_expression": "assets" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._respectful_pause", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 1021, "line_end": 1024, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 1022, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 145, "line_end": 164, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_collection", "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "preflight_crawl" ], "literal_get_calls": [ { "line": 148, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 150, "receiver": "preflight", "key": "ok", "expression": "preflight.get('ok')" }, { "line": 151, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" }, { "line": 152, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" } ], "html_queries": [], "dictionary_entries": [ { "line": 149, "key": "collection", "value_expression": "','.join(collections)" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.preflight_crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 122, "line_end": 143, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 143, "key": "ok", "value_expression": "True" }, { "line": 125, "key": "ok", "value_expression": "False" }, { "line": 125, "key": "message", "value_expression": "'No supported digital sheet music collections were selected'" }, { "line": 140, "key": "ok", "value_expression": "False" }, { "line": 141, "key": "message", "value_expression": "'musescore_direct requires explicit URLs; an empty credential boundary is not a completed crawl'" }, { "line": 129, "key": "ok", "value_expression": "False" }, { "line": 129, "key": "message", "value_expression": "\"pdmx_mode must be either 'rows' or 'inventory'\"" }, { "line": 132, "key": "ok", "value_expression": "False" }, { "line": 134, "key": "message", "value_expression": "'PDMX row acquisition requires an existing pdmx_csv_path; use pdmx_mode=inventory only for an explicit dataset-inventory scope'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 166, "line_end": 179, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 167, "receiver": "raw_data", "key": "score_id", "expression": "raw_data.get('score_id')" }, { "line": 167, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 169, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 178, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "score_id", "value_expression": "raw_data.get('score_id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "id", "value_expression": "raw_data.get('id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 169, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "transform_run_id", "value_expression": "raw_data.get('transform_run_id')", "line": 178, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 148, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "ok", "value_expression": "preflight.get('ok')", "line": 150, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 172, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 174, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:\n pass\n return normalize_digital_sheet_music_record(" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 175, "line_end": 179, "text": " payload,\n primary_id=score_id,\n raw_content=json.dumps(payload, ensure_ascii=False, sort_keys=True),\n transform_run_id=raw_data.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 145, "line_end": 153, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 147, "line_end": 155, "text": " collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))\n records: List[Dict[str, Any]] = []\n for index, collection in enumerate(collections, start=1):" } ] } }, { "catalogue_id": "S008", "source_key": "estey", "name": "Estey Pipe Organ Archive", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Matrix Associates / The Estey Organ virtual museum", "scope_description": "Estey company history, instruments and documentary material", "documentary_unit": "Opus-list entry or archival item", "source_description": "The virtual museum joins a pipe-organ opus list to catalogues, photographs, facade drawings, patents, recordings and company history. Although now served from the Pipe Organ Database archive host, the page identifies itself as a privately owned website and carries a Matrix Associates copyright notice.", "source_description_references": [ "estey", "record_estey" ], "interpretation": "Cite the original archive and individual item as well as the current host. Manufacturing totals in the introduction are historical company statements, not counts of surviving instruments or digitised records.", "rights_observation": "The page invites downloading photographs and information and requests attribution when they are reused; it does not name a standard open licence.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "estey", "record_estey" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_estey", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Static opus record combining stop table and transcribed historical notice", "repeated_information": "Manual and pedal groups followed by couplers and action details.", "date_and_state_distinctions": "The dated newspaper notice is a documentary account, not an update date for the website.", "interpretive_limit": "Keep the opus specification separate from the historical notice and its publication date.", "observations": [ "Church, town and state identify the installation.", "Estey opus identity is supplied by the record and archive context.", "The historical notice is dated to the instrument opening.", "Named stops and pitches grouped under Great, Swell and Pedal.", "Pedal compass and division labels are given.", "Tubular-pneumatic action is stated.", "Couplers and swell control are listed.", "The page incorporates a dated historical press account." ], "uninspected_or_unfilled_material": [], "narrative": "Opus 1 combines a manual-and-pedal specification with a transcribed historical press notice. Couplers and tubular-pneumatic action are described alongside the stoplist. The notice has its own documentary date; it is not a date of revision for the website. The opus record and the historical account should remain distinguishable when citing the installation." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The opus index and individual archive pages have separate parsers. Detail titles and specifications receive archive-specific treatment rather than relying entirely on the generic static-page routines.", "transformation": "The specialised specification extractor reads manual/pedal sections of the archive layout; the remainder of the transfer uses the shared location, metadata, media and source-text machinery. The documentary unit should remain the opus/installation account, with its press notices or other historical documents separately attributed.", "limitations": "A factory stoplist, later alteration and transcribed notice can have different dates. The adapter does not settle continuity between an opus and a surviving instrument. Archive-specific stop extraction does not justify treating all inherited generic event rules as historically validated." }, "implementation": { "class": "EsteyArchiveCrawler", "class_line": 1079, "path": "aggregator/static_html_crawler.py", "inheritance": [ "EsteyArchiveCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "estey", "path": "aggregator/static_html_crawler.py", "line": 1080, "class": "EsteyArchiveCrawler" }, "id_column": { "value": "estey_id", "path": "aggregator/static_html_crawler.py", "line": 1081, "class": "EsteyArchiveCrawler" }, "default_country": { "value": "United States", "path": "aggregator/static_html_crawler.py", "line": 1084, "class": "EsteyArchiveCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://archive.pipeorgandatabase.org/estey/opuslist.html" ], "path": "aggregator/static_html_crawler.py", "line": 1085, "class": "EsteyArchiveCrawler" }, "detail_url_patterns": { "value": [ "/estey/opus" ], "path": "aggregator/static_html_crawler.py", "line": 1086, "class": "EsteyArchiveCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://archive.pipeorgandatabase.org/estey", "path": "aggregator/static_html_crawler.py", "line": 1083, "class": "EsteyArchiveCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "EsteyArchiveCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 1088, "line_end": 1138, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_clean_text", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 1128, "expression": "soup.find_all('table')" }, { "line": 1129, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "EsteyArchiveCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 1160, "line_end": 1163, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "EsteyArchiveCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 1165, "line_end": 1169, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "EsteyArchiveCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 1140, "line_end": 1158, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_reference_from_parts" ], "literal_get_calls": [], "html_queries": [ { "line": 1143, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 1155, "key": "opus", "value_expression": "opus" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 434, "line_end": 516, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_extract_images", "_extract_links", "_extract_metadata", "_extract_raw_spec", "_extract_table_rows", "_extract_title", "_first_int", "_first_metadata", "_first_year", "_location_parts", "_main_text", "_safe_json_loads" ], "literal_get_calls": [ { "line": 435, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 440, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 441, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 442, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 443, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 444, "receiver": "raw_data", "key": "rank_count", "expression": "raw_data.get('rank_count')" }, { "line": 513, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 442, "receiver": "metadata", "key": "manuals", "expression": "metadata.get('manuals')" }, { "line": 442, "receiver": "metadata", "key": "manual", "expression": "metadata.get('manual')" }, { "line": 443, "receiver": "metadata", "key": "stops", "expression": "metadata.get('stops')" }, { "line": 443, "receiver": "metadata", "key": "stop", "expression": "metadata.get('stop')" }, { "line": 444, "receiver": "metadata", "key": "ranks", "expression": "metadata.get('ranks')" }, { "line": 444, "receiver": "metadata", "key": "rank", "expression": "metadata.get('rank')" }, { "line": 480, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 453, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 454, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 479, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 487, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': raw_data.get('country') or self.default_country, 'region': raw_data.get('region') or self.default_region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 460, "key": "involved", "value_expression": "self._build_involved(builder, year)" }, { "line": 461, "key": "events", "value_expression": "[{'type': 'construction', 'date': str(year), 'description': builder}] if year else []" }, { "line": 462, "key": "specifications", "value_expression": "{'manual_count': manual_count, 'stop_count': stop_count, 'rank_count': rank_count, 'stops_by_division': stops_by_division, 'stops': [{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops], 'raw_spec': raw_spec, 'technical_rows_unstructured': [{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows}" }, { "line": 479, "key": "media", "value_expression": "{'images': self._extract_images(soup, raw_data.get('url') or '')}" }, { "line": 480, "key": "links", "value_expression": "self._extract_links(soup, raw_data.get('url') or '')" }, { "line": 481, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': text}] if text else []" }, { "line": 482, "key": "other_identifiers", "value_expression": "[{'type': self.id_column, 'value': raw_data.get(self.id_column)}]" }, { "line": 483, "key": "extensions", "value_expression": "{self.source_key: {'title': title, 'metadata': metadata, 'raw_data': self._safe_json_loads(raw_data.get('raw_data_json'), {}), 'reference_context': {key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}}}" }, { "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 467, "key": "stops", "value_expression": "[{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 472, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 473, "key": "technical_rows_unstructured", "value_expression": "[{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows" }, { "line": 479, "key": "images", "value_expression": "self._extract_images(soup, raw_data.get('url') or '')" }, { "line": 452, "key": "status", "value_expression": "'current_location'" }, { "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "line": 454, "key": "region", "value_expression": "raw_data.get('region') or self.default_region" }, { "line": 455, "key": "town", "value_expression": "town" }, { "line": 456, "key": "building", "value_expression": "building" }, { "line": 457, "key": "raw_str", "value_expression": "title" }, { "line": 482, "key": "type", "value_expression": "self.id_column" }, { "line": 482, "key": "value", "value_expression": "raw_data.get(self.id_column)" }, { "line": 485, "key": "title", "value_expression": "title" }, { "line": 486, "key": "metadata", "value_expression": "metadata" }, { "line": 487, "key": "raw_data", "value_expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {})" }, { "line": 488, "key": "reference_context", "value_expression": "{key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}" }, { "line": 461, "key": "type", "value_expression": "'construction'" }, { "line": 461, "key": "date", "value_expression": "str(year)" }, { "line": 461, "key": "description", "value_expression": "builder" }, { "line": 468, "key": "name", "value_expression": "stop" }, { "line": 468, "key": "division", "value_expression": "division" }, { "line": 468, "key": "raw_str", "value_expression": "stop" }, { "line": 481, "key": "type", "value_expression": "'source_text'" }, { "line": 481, "key": "text", "value_expression": "text" }, { "line": 474, "key": "label", "value_expression": "key" }, { "line": 474, "key": "value", "value_expression": "value" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 457, "key": "raw_str", "value_expression": "title" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 461, "line_end": 469, "text": " \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 460, "line_end": 468, "text": " \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 462, "line_end": 470, "text": " \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 463, "line_end": 471, "text": " \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 450, "line_end": 458, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": raw_data.get(\"country\") or self.default_country,\n \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 454, "line_end": 462, "text": " \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }\n ],\n \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {" } ] } }, { "catalogue_id": "S009", "source_key": "evensong_hauptwerk_samplesets", "name": "Evensong Hauptwerk Sample Sets", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Evensong LLC", "scope_description": "Selected instruments, prominently North American organs", "documentary_unit": "Hauptwerk sample-set product", "source_description": "The catalogue pairs named organs with product pages and describes the producer's sampling history. It acknowledges extensions using stops from other sources in an early product. A separate sample-set licence is linked from the catalogue.", "source_description_references": [ "evensong_hauptwerk_samplesets", "evensong_rights" ], "interpretation": "A product's virtual specification may exceed its recorded instrument. The package version and the original or extended configuration must therefore accompany comparisons of stops and manuals.", "rights_observation": "A product licence governs use. Its repurposing clause is conditional and should be read for the specific release; no blanket CC0 permission is recorded.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "evensong_hauptwerk_samplesets", "evensong_rights" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects WordPress page links, captioned figures and linked images. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is linked product entries with image/title context. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A linked figure can be promotional navigation. Product identity and represented instrument identity require separate evidence. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "EvensongHauptwerkSamplesetsCrawler", "class_line": 1522, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "EvensongHauptwerkSamplesetsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "evensong_hauptwerk_samplesets", "path": "aggregator/grandorgue_library_crawler.py", "line": 1523, "class": "EvensongHauptwerkSamplesetsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 1525, "class": "EvensongHauptwerkSamplesetsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1527, "line_end": 1651, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1537, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" }, { "line": 1584, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 1578, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 1578, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 1573, "expression": "soup.select('a[href*=\"page_id=\"]')" }, { "line": 1532, "expression": "soup.select('figure.wp-caption')" }, { "line": 1534, "expression": "figure.select_one('a[href*=\"page_id=\"]')" }, { "line": 1577, "expression": "link.select_one('img')" } ], "dictionary_entries": [ { "line": 1544, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Samplesets for Hauptwerk'}" }, { "line": 1545, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Evensong', 'description_excerpt': _excerpt(_clean_text(figure.get_text(' ', strip=True))), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'commercial', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1554, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1560, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1561, "key": "extensions", "value_expression": "{'source_fragment_html': str(figure)}" }, { "line": 1594, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Samplesets for Hauptwerk'}" }, { "line": 1595, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Evensong', 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'commercial', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1604, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1610, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1611, "key": "extensions", "value_expression": "{'source_fragment_html': str(link)}" }, { "line": 1629, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Samplesets for Hauptwerk'}" }, { "line": 1630, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Evensong', 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'commercial', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1639, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1640, "key": "links", "value_expression": "[{'url': source_url, 'link_text': 'Samplesets for Hauptwerk', 'is_product_homepage_candidate': True}]" }, { "line": 1641, "key": "extensions", "value_expression": "{'source_fragment_html': title}" }, { "line": 1544, "key": "position", "value_expression": "position" }, { "line": 1544, "key": "section_label", "value_expression": "'Samplesets for Hauptwerk'" }, { "line": 1546, "key": "title", "value_expression": "title" }, { "line": 1547, "key": "producer_label", "value_expression": "'Evensong'" }, { "line": 1548, "key": "description_excerpt", "value_expression": "_excerpt(_clean_text(figure.get_text(' ', strip=True)))" }, { "line": 1549, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1550, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1551, "key": "access_model_hint", "value_expression": "'commercial'" }, { "line": 1552, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1555, "key": "package_title", "value_expression": "title" }, { "line": 1556, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1557, "key": "is_odf_only", "value_expression": "False" }, { "line": 1558, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1561, "key": "source_fragment_html", "value_expression": "str(figure)" }, { "line": 1594, "key": "position", "value_expression": "position" }, { "line": 1594, "key": "section_label", "value_expression": "'Samplesets for Hauptwerk'" }, { "line": 1596, "key": "title", "value_expression": "title" }, { "line": 1597, "key": "producer_label", "value_expression": "'Evensong'" }, { "line": 1598, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1599, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1600, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1601, "key": "access_model_hint", "value_expression": "'commercial'" }, { "line": 1602, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1605, "key": "package_title", "value_expression": "title" }, { "line": 1606, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1607, "key": "is_odf_only", "value_expression": "False" }, { "line": 1608, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1611, "key": "source_fragment_html", "value_expression": "str(link)" }, { "line": 1629, "key": "position", "value_expression": "position" }, { "line": 1629, "key": "section_label", "value_expression": "'Samplesets for Hauptwerk'" }, { "line": 1631, "key": "title", "value_expression": "title" }, { "line": 1632, "key": "producer_label", "value_expression": "'Evensong'" }, { "line": 1633, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1634, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1635, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1636, "key": "access_model_hint", "value_expression": "'commercial'" }, { "line": 1637, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1639, "key": "package_title", "value_expression": "title" }, { "line": 1639, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1639, "key": "is_odf_only", "value_expression": "False" }, { "line": 1639, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1641, "key": "source_fragment_html", "value_expression": "title" }, { "line": 1560, "key": "url", "value_expression": "detail_url" }, { "line": 1560, "key": "link_text", "value_expression": "title" }, { "line": 1560, "key": "is_product_homepage_candidate", "value_expression": "True" }, { "line": 1610, "key": "url", "value_expression": "detail_url" }, { "line": 1610, "key": "link_text", "value_expression": "title" }, { "line": 1610, "key": "is_product_homepage_candidate", "value_expression": "True" }, { "line": 1640, "key": "url", "value_expression": "source_url" }, { "line": 1640, "key": "link_text", "value_expression": "'Samplesets for Hauptwerk'" }, { "line": 1640, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1556, "key": "package_kind", "value_expression": "'sample_set'" }, { "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1558, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1557, "key": "is_odf_only", "value_expression": "False" }, { "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1547, "key": "producer_label", "value_expression": "'Evensong'" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "line_start": 1553, "line_end": 1561, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(figure)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "line_start": 1555, "line_end": 1563, "text": " \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(figure)},\n },\n primary_id=slugify(detail_url)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "line_start": 1554, "line_end": 1562, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(figure)},\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "EvensongHauptwerkSamplesetsCrawler.parse_html", "line_start": 1544, "line_end": 1552, "text": " \"catalog\": {\"position\": position, \"section_label\": \"Samplesets for Hauptwerk\"},\n \"item\": {\n \"title\": title,\n \"producer_label\": \"Evensong\",\n \"description_excerpt\": _excerpt(_clean_text(figure.get_text(\" \", strip=True))),\n \"software_labels\": [\"Hauptwerk\"],\n \"engine_labels\": [\"Hauptwerk\"],\n \"access_model_hint\": \"commercial\",\n \"availability_status_hint\": \"producer_catalog_entry\"," } ] } }, { "catalogue_id": "S010", "source_key": "gdo_literature", "name": "GDO Literaturdatenbank", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Alfred Reichling; Matthias Reichling; Gesellschaft der Orgelfreunde", "scope_description": "Literature on organ building and playing; scores excluded", "documentary_unit": "Bibliographic entry", "source_description": "The society credits Alfred and Matthias Reichling with the bibliography. The introduction describes journal surveys, monographs, commemorative volumes, proceedings and articles, with author, title, publication details and annotations. The configured database endpoint returned HTTP 500 during this check; the explanatory page remained readable.", "source_description_references": [ "gdo_intro", "gdo_literature" ], "interpretation": "The bibliography is a route to publications, not evidence that their full texts were consulted.", "rights_observation": "No general licence for redistributing the complete bibliography was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "gdo_intro", "gdo_literature" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler acquires bibliographic result rows and associated query context. Cached structured payload and row fields are merged before normalization.", "transformation": "The normalizer preserves the citation text and original author label while separately splitting authors, identifying publication years and constructing title, publication-detail and annotation fields. Content-derived identifiers are used when no stable native identifier is supplied. The source page remains the provenance for a bibliographic entry, not the cited publication itself.", "limitations": "A year extracted from a citation is not necessarily a single publication date; series, volumes and cited historical dates may coexist. Content-hash identity can change with editorial spelling. A catalogue citation is not evidence that the underlying publication was consulted." }, "implementation": { "class": "GDOLiteratureCrawler", "class_line": 25, "path": "aggregator/gdo_literature_crawler.py", "inheritance": [ "GDOLiteratureCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "literature_id", "path": "aggregator/gdo_literature_crawler.py", "line": 33, "class": "GDOLiteratureCrawler" }, "record_label": { "value": "literature records", "path": "aggregator/gdo_literature_crawler.py", "line": 34, "class": "GDOLiteratureCrawler" } }, "methods": [ { "symbol": "GDOLiteratureCrawler._crawl_options", "path": "aggregator/gdo_literature_crawler.py", "line_start": 440, "line_end": 452, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_years_from_options" ], "literal_get_calls": [ { "line": 447, "receiver": "source", "key": "year", "expression": "source.get('year')" }, { "line": 447, "receiver": "source", "key": "years", "expression": "source.get('years')" } ], "html_queries": [], "dictionary_entries": [ { "line": 449, "key": "mode", "value_expression": "mode" }, { "line": 450, "key": "years", "value_expression": "years" }, { "line": 451, "key": "declared_total_records", "value_expression": "None" } ], "subscript_assignments": [ { "line": 445, "value_line": 445, "key": "year", "target_expression": "source['year']", "value_expression": "target" } ] }, { "symbol": "GDOLiteratureCrawler._parse_years", "path": "aggregator/gdo_literature_crawler.py", "line_start": 467, "line_end": 485, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler._publication_details", "path": "aggregator/gdo_literature_crawler.py", "line_start": 433, "line_end": 438, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 435, "expression": "clone.find_all(['i', 'small'])" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler._record_from_nodes", "path": "aggregator/gdo_literature_crawler.py", "line_start": 393, "line_end": 431, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_clean_text", "_publication_details" ], "literal_get_calls": [ { "line": 410, "receiver": "query", "key": "suchjahr", "expression": "query.get('suchjahr')" } ], "html_queries": [ { "line": 403, "expression": "dd.find('i')" }, { "line": 405, "expression": "dd.find_all('small')" } ], "dictionary_entries": [ { "line": 413, "key": "literature_id", "value_expression": "f'gdo-lit-{content_hash[:16]}'" }, { "line": 414, "key": "content_sha256", "value_expression": "content_hash" }, { "line": 415, "key": "query_year", "value_expression": "year_query" }, { "line": 416, "key": "author", "value_expression": "author" }, { "line": 417, "key": "title", "value_expression": "title" }, { "line": 418, "key": "citation_text", "value_expression": "citation_text" }, { "line": 419, "key": "publication_details", "value_expression": "publication_details" }, { "line": 420, "key": "publication_years", "value_expression": "sorted(set(re.findall('\\\\b(?:1[5-9]\\\\d{2}|20\\\\d{2})\\\\b', citation_text)))" }, { "line": 421, "key": "notes", "value_expression": "notes" }, { "line": 422, "key": "url", "value_expression": "SOURCE_DEFINITION['base_url']" }, { "line": 423, "key": "source_url", "value_expression": "SOURCE_DEFINITION['base_url']" }, { "line": 424, "key": "status", "value_expression": "'initiated'" }, { "line": 425, "key": "raw_html", "value_expression": "raw_html" }, { "line": 426, "key": "query", "value_expression": "dict(query)" }, { "line": 427, "key": "result_index", "value_expression": "result_index" }, { "line": 428, "key": "declared_total_records", "value_expression": "declared_total" }, { "line": 429, "key": "year_result_count", "value_expression": "found_count" }, { "line": 430, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" } ], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler.crawl", "path": "aggregator/gdo_literature_crawler.py", "line_start": 83, "line_end": 130, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_crawl_options", "_persist_cache_records", "_upsert_scrape_status", "_year_query_delay", "fetch_declared_total_records", "is_scrape_stop_requested", "mark_scrape_stopped", "search_year" ], "literal_get_calls": [ { "line": 85, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 93, "value_line": 93, "key": "declared_total_records", "target_expression": "options['declared_total_records']", "value_expression": "declared_total" }, { "line": 112, "value_line": 112, "key": "declared_total_records", "target_expression": "record['declared_total_records']", "value_expression": "declared_total" }, { "line": 113, "value_line": 113, "key": "year_result_count", "target_expression": "record['year_result_count']", "value_expression": "found_count" } ] }, { "symbol": "GDOLiteratureCrawler.fetch_declared_total_records", "path": "aggregator/gdo_literature_crawler.py", "line_start": 145, "line_end": 153, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_declared_total_from_html" ], "literal_get_calls": [ { "line": 147, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler.normalized_batch_record", "path": "aggregator/gdo_literature_crawler.py", "line_start": 277, "line_end": 296, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 290, "value_line": 290, "key": "transform_run_id", "target_expression": "identity['transform_run_id']", "value_expression": "process_id" }, { "line": 291, "value_line": 291, "key": "artifact_ids", "target_expression": "identity['artifact_ids']", "value_expression": "list(artifact_ids or [])" }, { "line": 293, "value_line": 293, "key": "url", "target_expression": "record.setdefault('source_snapshot', {})['url']", "value_expression": "source_url" }, { "line": 295, "value_line": 295, "key": "retrieved_at", "target_expression": "record.setdefault('source', {})['retrieved_at']", "value_expression": "retrieved_at" } ] }, { "symbol": "GDOLiteratureCrawler.parse_results_html", "path": "aggregator/gdo_literature_crawler.py", "line_start": 155, "line_end": 185, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_clean_author", "_declared_total_from_html", "_found_count_from_html", "_record_from_nodes" ], "literal_get_calls": [], "html_queries": [ { "line": 159, "expression": "soup.find('dl')" }, { "line": 166, "expression": "dl.find_all(['dt', 'dd'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler.preflight_crawl", "path": "aggregator/gdo_literature_crawler.py", "line_start": 77, "line_end": 81, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [ "_years_from_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 81, "key": "ok", "value_expression": "True" }, { "line": 81, "key": "years", "value_expression": "years[:5]" }, { "line": 81, "key": "year_count", "value_expression": "len(years)" }, { "line": 80, "key": "ok", "value_expression": "False" }, { "line": 80, "key": "message", "value_expression": "'No valid year query could be built.'" } ], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler.scrape_row_to_transfer_record", "path": "aggregator/gdo_literature_crawler.py", "line_start": 259, "line_end": 275, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [], "literal_get_calls": [ { "line": 268, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 272, "receiver": "raw_payload", "key": "raw_html", "expression": "raw_payload.get('raw_html')" }, { "line": 272, "receiver": "raw_payload", "key": "citation_text", "expression": "raw_payload.get('citation_text')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOLiteratureCrawler.transform", "path": "aggregator/gdo_literature_crawler.py", "line_start": 187, "line_end": 203, "inherited": false, "file_sha256": "0acff63fc05fb9ebb8e24b440755431d13ecc3bc24b800875c1db0f6b30856fa", "self_calls": [], "literal_get_calls": [ { "line": 189, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 201, "receiver": "payload", "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 202, "receiver": "payload", "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 200, "receiver": "payload", "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 200, "receiver": "payload", "key": "citation_text", "expression": "payload.get('citation_text')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GDOLiteratureCrawler.transform", "path": "aggregator/gdo_literature_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 189, "kind": "input read" }, { "symbol": "GDOLiteratureCrawler.transform", "path": "aggregator/gdo_literature_crawler.py", "key": "crawl_run_id", "value_expression": "payload.get('crawl_run_id')", "line": 201, "kind": "input read" }, { "symbol": "GDOLiteratureCrawler.transform", "path": "aggregator/gdo_literature_crawler.py", "key": "transform_run_id", "value_expression": "payload.get('transform_run_id')", "line": 202, "kind": "input read" }, { "symbol": "GDOLiteratureCrawler.transform", "path": "aggregator/gdo_literature_crawler.py", "key": "raw_html", "value_expression": "payload.get('raw_html')", "line": 200, "kind": "input read" }, { "symbol": "GDOLiteratureCrawler.transform", "path": "aggregator/gdo_literature_crawler.py", "key": "citation_text", "value_expression": "payload.get('citation_text')", "line": 200, "kind": "input read" }, { "symbol": "GDOLiteratureCrawler.crawl", "path": "aggregator/gdo_literature_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 85, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/gdo_literature_crawler.py", "symbol": "GDOLiteratureCrawler.transform", "line_start": 187, "line_end": 194, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n payload = dict(raw_data or {})\n raw_json = payload.get(\"raw_data_json\")\n if isinstance(raw_json, str) and raw_json.strip():\n try:\n decoded = json.loads(raw_json)\n if isinstance(decoded, dict):\n decoded.update({key: value for key, value in payload.items() if value not in (None, \"\")})" }, { "path": "aggregator/gdo_literature_crawler.py", "symbol": "GDOLiteratureCrawler.transform", "line_start": 198, "line_end": 203, "text": " return normalize_gdo_literature_record(\n payload,\n raw_content=payload.get(\"raw_html\") or payload.get(\"citation_text\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/gdo_literature_crawler.py", "symbol": "GDOLiteratureCrawler.transform", "line_start": 199, "line_end": 203, "text": " payload,\n raw_content=payload.get(\"raw_html\") or payload.get(\"citation_text\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/gdo_literature_crawler.py", "symbol": "GDOLiteratureCrawler.transform", "line_start": 197, "line_end": 203, "text": " pass\n return normalize_gdo_literature_record(\n payload,\n raw_content=payload.get(\"raw_html\") or payload.get(\"citation_text\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/gdo_literature_crawler.py", "symbol": "GDOLiteratureCrawler.crawl", "line_start": 83, "line_end": 90, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n task_id = kwargs.get(\"task_id\")\n years = options[\"years\"]\n total_steps = max(1, len(years))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting GDO literature crawl.\", 0, total_steps)\n\n records_by_id: Dict[str, Dict[str, Any]] = {}" } ] } }, { "catalogue_id": "S011", "source_key": "gdo_orgelbauer_links", "name": "GdO Orgelbauer Links", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Gesellschaft der Orgelfreunde", "scope_description": "International organ-building directory grouped by country", "documentary_unit": "Directory link to a builder, supplier or association", "source_description": "The directory includes both individual firms and organ-building associations. Country headings organise the links. A firm's name and destination website form useful identifying evidence, while association links describe a different type of organisation.", "source_description_references": [ "gdo_orgelbauer_links" ], "interpretation": "A directory appearance does not establish current trading status, legal continuity or responsibility for a historical instrument. Those questions require the destination's own records and dated construction evidence.", "rights_observation": "The directory carries a society copyright notice; no general open licence was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "gdo_orgelbauer_links" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler treats an outbound directory link and its heading path as the acquisition unit. Link text, directory URL and target website are preserved separately.", "transformation": "Normalization constructs an institution-directory entry, retains display label, country, locality hint and heading position, and supplies a derived identifier when necessary. The linked institution site is recorded as a candidate for later enrichment. That intended later work is not already part of the directory evidence.", "limitations": "Directory grouping does not prove an institution's legal identity, foundation date or present operation. A website URL and an organisation are different entities. Default organ-builder classification follows the directory context and should not silently become an independently verified occupation." }, "implementation": { "class": "GDOOrgelbauerLinksCrawler", "class_line": 30, "path": "aggregator/gdo_orgelbauer_links_crawler.py", "inheritance": [ "GDOOrgelbauerLinksCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "institution_id", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line": 38, "class": "GDOOrgelbauerLinksCrawler" }, "record_label": { "value": "institution records", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line": 39, "class": "GDOOrgelbauerLinksCrawler" } }, "methods": [ { "symbol": "GDOOrgelbauerLinksCrawler._fetch_html", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 134, "line_end": 137, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [], "literal_get_calls": [ { "line": 135, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOOrgelbauerLinksCrawler._normalize_url", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 501, "line_end": 507, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOOrgelbauerLinksCrawler.crawl", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 113, "line_end": 132, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [ "_fetch_html", "_persist_cache_records", "_upsert_scrape_status", "parse_directory_html" ], "literal_get_calls": [ { "line": 114, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 115, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOOrgelbauerLinksCrawler.normalized_batch_record", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 274, "line_end": 295, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 287, "value_line": 287, "key": "transform_run_id", "target_expression": "identity['transform_run_id']", "value_expression": "process_id" }, { "line": 288, "value_line": 288, "key": "artifact_ids", "target_expression": "identity['artifact_ids']", "value_expression": "list(artifact_ids or [])" }, { "line": 291, "value_line": 291, "key": "url", "target_expression": "source_snapshot['url']", "value_expression": "source_url" }, { "line": 292, "value_line": 292, "key": "source_url", "target_expression": "source_snapshot['source_url']", "value_expression": "source_url" }, { "line": 294, "value_line": 294, "key": "retrieved_at", "target_expression": "record.setdefault('source', {})['retrieved_at']", "value_expression": "retrieved_at" } ] }, { "symbol": "GDOOrgelbauerLinksCrawler.parse_directory_html", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 139, "line_end": 185, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [ "_classify_institution", "_clean_text", "_context_for_link", "_is_outbound_institution_url", "_normalize_url", "_split_name_locality" ], "literal_get_calls": [ { "line": 147, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 144, "expression": "content.find_all('a', href=True)" }, { "line": 141, "expression": "soup.find(id='c3075')" }, { "line": 141, "expression": "soup.find('main')" } ], "dictionary_entries": [ { "line": 165, "key": "institution_id", "value_expression": "institution_id" }, { "line": 166, "key": "content_sha256", "value_expression": "content_hash" }, { "line": 167, "key": "name", "value_expression": "name" }, { "line": 168, "key": "link_text", "value_expression": "link_text" }, { "line": 169, "key": "institution_type", "value_expression": "self._classify_institution(link_text)" }, { "line": 170, "key": "country", "value_expression": "country" }, { "line": 171, "key": "locality_hint", "value_expression": "locality_hint" }, { "line": 172, "key": "website_url", "value_expression": "normalized_url" }, { "line": 173, "key": "website_host", "value_expression": "urlsplit(normalized_url).netloc.lower()" }, { "line": 174, "key": "directory_url", "value_expression": "page_url" }, { "line": 175, "key": "source_url", "value_expression": "page_url" }, { "line": 176, "key": "section_id", "value_expression": "section_id" }, { "line": 177, "key": "heading_path", "value_expression": "heading_path" }, { "line": 178, "key": "position", "value_expression": "len(records) + 1" }, { "line": 179, "key": "status", "value_expression": "'initiated'" }, { "line": 180, "key": "raw_html", "value_expression": "str(link)" }, { "line": 181, "key": "raw_link_html", "value_expression": "str(link)" }, { "line": 182, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" } ], "subscript_assignments": [] }, { "symbol": "GDOOrgelbauerLinksCrawler.preflight_crawl", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 110, "line_end": 111, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [], "literal_get_calls": [ { "line": 111, "receiver": "task_kwargs", "key": "url", "expression": "task_kwargs.get('url')" } ], "html_queries": [], "dictionary_entries": [ { "line": 111, "key": "ok", "value_expression": "True" }, { "line": 111, "key": "url", "value_expression": "str(task_kwargs.get('url') or SOURCE_DEFINITION['base_url'])" } ], "subscript_assignments": [] }, { "symbol": "GDOOrgelbauerLinksCrawler.scrape_row_to_transfer_record", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 255, "line_end": 272, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [], "literal_get_calls": [ { "line": 264, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 269, "receiver": "raw_payload", "key": "raw_html", "expression": "raw_payload.get('raw_html')" }, { "line": 269, "receiver": "raw_payload", "key": "raw_link_html", "expression": "raw_payload.get('raw_link_html')" }, { "line": 269, "receiver": "raw_payload", "key": "link_text", "expression": "raw_payload.get('link_text')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GDOOrgelbauerLinksCrawler.transform", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line_start": 187, "line_end": 211, "inherited": false, "file_sha256": "30f4304bde50591faeaad6b6beb79421129cd4a26bef4800a299dc1810300c5d", "self_calls": [], "literal_get_calls": [ { "line": 189, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 198, "receiver": "payload", "key": "heading_path", "expression": "payload.get('heading_path')" }, { "line": 209, "receiver": "payload", "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 210, "receiver": "payload", "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 208, "receiver": "payload", "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 208, "receiver": "payload", "key": "raw_link_html", "expression": "payload.get('raw_link_html')" }, { "line": 208, "receiver": "payload", "key": "link_text", "expression": "payload.get('link_text')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 202, "value_line": 202, "key": "heading_path", "target_expression": "payload['heading_path']", "value_expression": "heading_path" }, { "line": 204, "value_line": 204, "key": "heading_path", "target_expression": "payload['heading_path']", "value_expression": "[payload['heading_path']]" } ] } ], "selected_rule_rows": [ { "symbol": "GDOOrgelbauerLinksCrawler.parse_directory_html", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "line": 170, "key": "country", "value_expression": "country" }, { "symbol": "GDOOrgelbauerLinksCrawler.transform", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 189, "kind": "input read" }, { "symbol": "GDOOrgelbauerLinksCrawler.transform", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "key": "heading_path", "value_expression": "payload.get('heading_path')", "line": 198, "kind": "input read" }, { "symbol": "GDOOrgelbauerLinksCrawler.transform", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "key": "crawl_run_id", "value_expression": "payload.get('crawl_run_id')", "line": 209, "kind": "input read" }, { "symbol": "GDOOrgelbauerLinksCrawler.transform", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "key": "transform_run_id", "value_expression": "payload.get('transform_run_id')", "line": 210, "kind": "input read" }, { "symbol": "GDOOrgelbauerLinksCrawler.transform", "path": "aggregator/gdo_orgelbauer_links_crawler.py", "key": "raw_html", "value_expression": "payload.get('raw_html')", "line": 208, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/gdo_orgelbauer_links_crawler.py", "symbol": "GDOOrgelbauerLinksCrawler.parse_directory_html", "line_start": 167, "line_end": 175, "text": " \"name\": name,\n \"link_text\": link_text,\n \"institution_type\": self._classify_institution(link_text),\n \"country\": country,\n \"locality_hint\": locality_hint,\n \"website_url\": normalized_url,\n \"website_host\": urlsplit(normalized_url).netloc.lower(),\n \"directory_url\": page_url,\n \"source_url\": page_url," }, { "path": "aggregator/gdo_orgelbauer_links_crawler.py", "symbol": "GDOOrgelbauerLinksCrawler.transform", "line_start": 187, "line_end": 194, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n payload = dict(raw_data or {})\n raw_json = payload.get(\"raw_data_json\")\n if isinstance(raw_json, str) and raw_json.strip():\n try:\n decoded = json.loads(raw_json)\n if isinstance(decoded, dict):\n decoded.update({key: value for key, value in payload.items() if value not in (None, \"\")})" }, { "path": "aggregator/gdo_orgelbauer_links_crawler.py", "symbol": "GDOOrgelbauerLinksCrawler.transform", "line_start": 195, "line_end": 203, "text": " payload = decoded\n except json.JSONDecodeError:\n pass\n if isinstance(payload.get(\"heading_path\"), str):\n try:\n heading_path = json.loads(payload[\"heading_path\"])\n if isinstance(heading_path, list):\n payload[\"heading_path\"] = heading_path\n except json.JSONDecodeError:" }, { "path": "aggregator/gdo_orgelbauer_links_crawler.py", "symbol": "GDOOrgelbauerLinksCrawler.transform", "line_start": 206, "line_end": 211, "text": " payload,\n source_definition=SOURCE_DEFINITION,\n raw_content=payload.get(\"raw_html\") or payload.get(\"raw_link_html\") or payload.get(\"link_text\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/gdo_orgelbauer_links_crawler.py", "symbol": "GDOOrgelbauerLinksCrawler.transform", "line_start": 207, "line_end": 211, "text": " source_definition=SOURCE_DEFINITION,\n raw_content=payload.get(\"raw_html\") or payload.get(\"raw_link_html\") or payload.get(\"link_text\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/gdo_orgelbauer_links_crawler.py", "symbol": "GDOOrgelbauerLinksCrawler.transform", "line_start": 205, "line_end": 211, "text": " return normalize_institution_directory_record(\n payload,\n source_definition=SOURCE_DEFINITION,\n raw_content=payload.get(\"raw_html\") or payload.get(\"raw_link_html\") or payload.get(\"link_text\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" } ] } }, { "catalogue_id": "S012", "source_key": "grandorgue_github_wiki", "name": "GrandOrgue GitHub Wiki: Sample Sets", "selection_role": "registered_source", "parent_source_key": null, "attribution": "GrandOrgue project contributors", "scope_description": "GrandOrgue resources across multiple producers", "documentary_unit": "Sample-set or organ-definition-file directory entry", "source_description": "The wiki separates free or demonstration sample sets from organ definition files for commercial recordings. It names external hosts, including Lars Palo, MPS Orgelseite and Virtual Pipe Organs, and tells readers to consult individual licences.", "source_description_references": [ "grandorgue_github_wiki" ], "interpretation": "Several Aggregator keys describe resources also listed here. These are multiple discovery paths, not independent sample-set populations. A wiki revision is preferable to an undated link when recording changes in availability.", "rights_observation": "Individual sample-set and ODF licences apply; the software project's licence is not evidence of the audio's licence.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "grandorgue_github_wiki" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects wiki heading hierarchy and list/paragraph nodes inside the wiki or Markdown body. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is heading context and individual list entries. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Nested lists and headings can describe producers, licences or requirements rather than independent sample sets. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "GrandOrgueGitHubWikiCrawler", "class_line": 512, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "GrandOrgueGitHubWikiCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "grandorgue_github_wiki", "path": "aggregator/grandorgue_library_crawler.py", "line": 513, "class": "GrandOrgueGitHubWikiCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "registry", "path": "aggregator/grandorgue_library_crawler.py", "line": 515, "class": "GrandOrgueGitHubWikiCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 517, "line_end": 573, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [], "html_queries": [ { "line": 523, "expression": "scope.select('h1, h2, h3, li, p')" }, { "line": 519, "expression": "soup.select_one('#wiki-body')" }, { "line": 519, "expression": "soup.select_one('.markdown-body')" }, { "line": 534, "expression": "node.find('ul')" } ], "dictionary_entries": [ { "line": 549, "key": "catalog", "value_expression": "{'section_label': current_section, 'position': position}" }, { "line": 550, "key": "item", "value_expression": "{'title': title, 'description_excerpt': text, 'access_model_hint': 'free_or_demo' if 'free' in lower or 'demo' in lower else ''}" }, { "line": 555, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'odf' if is_odf else 'registry_entry', 'requires_sample_set': is_odf, 'is_odf_only': is_odf, 'requires_commercial_sample_set': is_odf and 'commercial' in lower}" }, { "line": 562, "key": "links", "value_expression": "links" }, { "line": 563, "key": "extensions", "value_expression": "{'source_fragment_html': str(node)}" }, { "line": 549, "key": "section_label", "value_expression": "current_section" }, { "line": 549, "key": "position", "value_expression": "position" }, { "line": 551, "key": "title", "value_expression": "title" }, { "line": 552, "key": "description_excerpt", "value_expression": "text" }, { "line": 553, "key": "access_model_hint", "value_expression": "'free_or_demo' if 'free' in lower or 'demo' in lower else ''" }, { "line": 556, "key": "package_title", "value_expression": "title" }, { "line": 557, "key": "package_kind", "value_expression": "'odf' if is_odf else 'registry_entry'" }, { "line": 558, "key": "requires_sample_set", "value_expression": "is_odf" }, { "line": 559, "key": "is_odf_only", "value_expression": "is_odf" }, { "line": 560, "key": "requires_commercial_sample_set", "value_expression": "is_odf and 'commercial' in lower" }, { "line": 563, "key": "source_fragment_html", "value_expression": "str(node)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 557, "key": "package_kind", "value_expression": "'odf' if is_odf else 'registry_entry'" }, { "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 558, "key": "requires_sample_set", "value_expression": "is_odf" }, { "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 559, "key": "is_odf_only", "value_expression": "is_odf" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "line_start": 554, "line_end": 562, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"odf\" if is_odf else \"registry_entry\",\n \"requires_sample_set\": is_odf,\n \"is_odf_only\": is_odf,\n \"requires_commercial_sample_set\": is_odf and \"commercial\" in lower,\n },\n \"links\": links," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "line_start": 555, "line_end": 563, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"odf\" if is_odf else \"registry_entry\",\n \"requires_sample_set\": is_odf,\n \"is_odf_only\": is_odf,\n \"requires_commercial_sample_set\": is_odf and \"commercial\" in lower,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueGitHubWikiCrawler.parse_html", "line_start": 556, "line_end": 564, "text": " \"package_title\": title,\n \"package_kind\": \"odf\" if is_odf else \"registry_entry\",\n \"requires_sample_set\": is_odf,\n \"is_odf_only\": is_odf,\n \"requires_commercial_sample_set\": is_odf and \"commercial\" in lower,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)},\n }," } ] } }, { "catalogue_id": "S013", "source_key": "indo", "name": "INdO", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Orgue en France", "scope_description": "France, including overseas departments and territories in the search interface", "documentary_unit": "Organ record with history and technical description", "source_description": "Orgue en France develops and maintains the collaborative national inventory, supported by the French Ministry of Culture since 2017. The site opened to public contributions in 2021 and describes its purpose as extending earlier departmental and regional inventories.", "source_description_references": [ "indo", "record_indo" ], "interpretation": "Existing and disappeared instruments are separate categories. Construction, rebuilding and current condition need their own dates and attributions. An inventory entry's completion state is distinct from the physical condition of the organ.", "rights_observation": "The consulted editorial page did not establish a single licence covering records, photographs and attachments.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "indo", "record_indo" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_indo", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Structured organ profile with linked authority records and contribution history", "repeated_information": "Sound divisions contain stops and compasses. Builder activities repeat with roles and dates.", "date_and_state_distinctions": "Construction and restoration dates are distinct from the page revision and contributor history.", "interpretive_limit": "Visible headings do not guarantee values. The examined barrel-organ record has no ordinary pedal division.", "observations": [ "Building, town, department and position within the building are populated.", "The organ profile links to a location map.", "Builder and restorer entries have roles and dates.", "Construction, restoration and modification activities are listed.", "Eight named stops with pitches in the selected manual division.", "Manual compass and note count are supplied.", "Key and stop actions and windchest description are populated.", "Temperament and pitch include a temperature-qualified measurement.", "The bellows/reservoir arrangement is described.", "Case and console have separate descriptions.", "Playability and ownership are stated, with heritage-protection identifiers.", "The record contains linked instrument images.", "Bibliographic sources and a linked documentary extract are identified.", "National heritage identifiers and builder-record links are present.", "Update date, named contributor and contribution history are displayed." ], "uninspected_or_unfilled_material": [], "narrative": "The Airvault barrel-organ record separates location, builder activities, technical description, sources and contribution history. It gives one manual division, a compass and note count, tuning information, a bellows description and heritage identifiers. It has no ordinary pedal division. That feature belongs to this instrument and cannot be interpreted as a missing-data rate for the national inventory." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The adapter acquires inventory detail pages and uses the cached town/building information as a fallback when the rendered heading does not supply it.", "transformation": "Composition cards are grouped by keyboard or pedal; their list items become source stop strings. Card headings determine the group names and a pedal-text test determines the branch. Location is read from the page heading and locality links. Timeline/history and media information are handled separately from the composition cards. Later offline attribution helpers operate on preserved evidence and are not evidence that the original crawler performed actor reconciliation.", "limitations": "A stop string can contain pitch and annotations without having those parts independently resolved. Keyboard grouping and the distinction between an instrument and successive works remain source-specific. The France value is supplied by the adapter. Neither that default nor a successful transform measures the national inventory's completeness." }, "implementation": { "class": "IndoCrawler", "class_line": 17, "path": "aggregator/indo_crawler.py", "inheritance": [ "IndoCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "IndoCrawler._fetch_page", "path": "aggregator/indo_crawler.py", "line_start": 216, "line_end": 234, "inherited": false, "file_sha256": "dd26703917e4a6ed86e523c05596b40727d5df9ec55e88bcf46b4430283b01d0", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 221, "key": "query", "value_expression": "''" }, { "line": 222, "key": "pageUrl", "value_expression": "f'?page={page}&sort=completion:desc'" }, { "line": 223, "key": "page", "value_expression": "page" }, { "line": 224, "key": "departement", "value_expression": "''" }, { "line": 225, "key": "sort", "value_expression": "'completion:desc'" }, { "line": 226, "key": "init", "value_expression": "'true' if page == 1 else 'false'" }, { "line": 229, "key": "Referer", "value_expression": "f'{self.base_url}/orgues/'" }, { "line": 230, "key": "X-Requested-With", "value_expression": "'XMLHttpRequest'" } ], "subscript_assignments": [] }, { "symbol": "IndoCrawler.crawl", "path": "aggregator/indo_crawler.py", "line_start": 47, "line_end": 193, "inherited": false, "file_sha256": "dd26703917e4a6ed86e523c05596b40727d5df9ec55e88bcf46b4430283b01d0", "self_calls": [ "_fetch_page", "_get_csrf_token", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 52, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 86, "receiver": "first_page_data", "key": "totalHits", "expression": "first_page_data.get('totalHits', 0)" }, { "line": 125, "receiver": "page_data", "key": "hits", "expression": "page_data.get('hits', [])" }, { "line": 84, "receiver": "first_page_data", "key": "totalPages", "expression": "first_page_data.get('totalPages', 1)" }, { "line": 95, "receiver": "kwargs", "key": "max_pages", "expression": "kwargs.get('max_pages')" }, { "line": 142, "receiver": "hit", "key": "id", "expression": "hit.get('id')" }, { "line": 143, "receiver": "hit", "key": "url", "expression": "hit.get('url')" }, { "line": 126, "receiver": "page_data", "key": "totalPages", "expression": "page_data.get('totalPages', declared_total_pages)" }, { "line": 152, "receiver": "hit", "key": "commune", "expression": "hit.get('commune', '')" }, { "line": 153, "receiver": "hit", "key": "edifice", "expression": "hit.get('edifice', '')" }, { "line": 154, "receiver": "hit", "key": "facteurs", "expression": "hit.get('facteurs', '')" }, { "line": 155, "receiver": "hit", "key": "construction", "expression": "hit.get('construction', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 150, "key": "indo_id", "value_expression": "indo_id" }, { "line": 151, "key": "country", "value_expression": "'France'" }, { "line": 152, "key": "town", "value_expression": "hit.get('commune', '')" }, { "line": 153, "key": "building", "value_expression": "hit.get('edifice', '')" }, { "line": 154, "key": "builder", "value_expression": "hit.get('facteurs', '')" }, { "line": 155, "key": "year", "value_expression": "str(hit.get('construction', ''))" }, { "line": 156, "key": "url", "value_expression": "full_url" }, { "line": 157, "key": "scraped_at", "value_expression": "datetime.now()" } ], "subscript_assignments": [] }, { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "line_start": 236, "line_end": 393, "inherited": false, "file_sha256": "dd26703917e4a6ed86e523c05596b40727d5df9ec55e88bcf46b4430283b01d0", "self_calls": [], "literal_get_calls": [ { "line": 240, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html', '')" }, { "line": 262, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town', '')" }, { "line": 369, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 259, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building', '')" }, { "line": 359, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 379, "receiver": "a_tag", "key": "title", "expression": "a_tag.get('title', '')" }, { "line": 389, "receiver": "raw_data", "key": "indo_id", "expression": "raw_data.get('indo_id')" }, { "line": 390, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" } ], "html_queries": [ { "line": 258, "expression": "soup.find('h2', class_='mb-0')" }, { "line": 261, "expression": "soup.find('h4', class_='text-muted')" }, { "line": 284, "expression": "soup.find_all('div', class_='clavier')" }, { "line": 332, "expression": "soup.find('dt', string='Facteurs')" }, { "line": 374, "expression": "soup.find_all('div', class_='portfolio-item')" }, { "line": 266, "expression": "sub_h4.find_all('a', class_='link')" }, { "line": 292, "expression": "clavier.find('div', class_='card-header')" }, { "line": 301, "expression": "clavier.find('ul', class_='list-group')" }, { "line": 376, "expression": "item.find('a', attrs={'data-lightbox': 'gallery-image'})" }, { "line": 303, "expression": "ul.find_all('li')" }, { "line": 336, "expression": "dd.find_all('li')" }, { "line": 293, "expression": "card_header.find('h4')" } ], "dictionary_entries": [ { "line": 273, "key": "country", "value_expression": "country" }, { "line": 274, "key": "town", "value_expression": "town" }, { "line": 275, "key": "building", "value_expression": "building" }, { "line": 276, "key": "status", "value_expression": "'current_location'" }, { "line": 314, "key": "id", "value_expression": "werk_id" }, { "line": 315, "key": "name", "value_expression": "name" }, { "line": 316, "key": "stops", "value_expression": "[]" }, { "line": 323, "key": "id", "value_expression": "werk_id" }, { "line": 324, "key": "name", "value_expression": "name" }, { "line": 376, "key": "data-lightbox", "value_expression": "'gallery-image'" }, { "line": 381, "key": "link", "value_expression": "img_url" }, { "line": 382, "key": "source", "value_expression": "title" }, { "line": 383, "key": "available", "value_expression": "True" }, { "line": 307, "key": "name", "value_expression": "stop_name" }, { "line": 308, "key": "raw_str", "value_expression": "stop_name" }, { "line": 364, "key": "name", "value_expression": "builder_name" }, { "line": 365, "key": "activities", "value_expression": "['builder']" }, { "line": 366, "key": "raw_str", "value_expression": "str(raw_data['builder'])" }, { "line": 346, "key": "name", "value_expression": "name" }, { "line": 347, "key": "notes", "value_expression": "[details]" }, { "line": 348, "key": "raw_str", "value_expression": "text" }, { "line": 352, "key": "name", "value_expression": "text" }, { "line": 353, "key": "raw_str", "value_expression": "text" } ], "subscript_assignments": [ { "line": 272, "value_line": 272, "key": "locations", "target_expression": "data['locations']", "value_expression": "[{'country': country, 'town': town, 'building': building, 'status': 'current_location'}]" }, { "line": 289, "value_line": 289, "key": "stops", "target_expression": "data['specifications']['stops']", "value_expression": "{}" }, { "line": 370, "value_line": 370, "key": "builder", "target_expression": "data.setdefault('extensions', {}).setdefault('indo', {}).setdefault('reference_context', {})['builder']", "value_expression": "str(raw_data['builder'])" } ] } ], "selected_rule_rows": [ { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "line": 273, "key": "country", "value_expression": "country" }, { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "line": 308, "key": "raw_str", "value_expression": "stop_name" }, { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "key": "html", "value_expression": "raw_data.get('html', '')", "line": 240, "kind": "input read" }, { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "key": "town", "value_expression": "raw_data.get('town', '')", "line": 262, "kind": "input read" }, { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "key": "builder", "value_expression": "raw_data.get('builder')", "line": 369, "kind": "input read" }, { "symbol": "IndoCrawler.transform", "path": "aggregator/indo_crawler.py", "key": "building", "value_expression": "raw_data.get('building', '')", "line": 259, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/indo_crawler.py", "symbol": "IndoCrawler.transform", "line_start": 270, "line_end": 278, "text": " \n # Locations\n data[\"locations\"] = [{\n \"country\": country,\n \"town\": town,\n \"building\": building,\n \"status\": \"current_location\"\n }]\n" }, { "path": "aggregator/indo_crawler.py", "symbol": "IndoCrawler.transform", "line_start": 305, "line_end": 313, "text": " stop_name = li.get_text(strip=True)\n stops_list.append({\n \"name\": stop_name,\n \"raw_str\": stop_name\n })\n \n if is_pedal:\n werk_id = f\"pedal-{ct_pedal}\"\n data[\"specifications\"][\"pedals\"].append({" }, { "path": "aggregator/indo_crawler.py", "symbol": "IndoCrawler.transform", "line_start": 237, "line_end": 245, "text": " \"\"\"\n Transforms the raw HTML into the standard JSON schema.\n \"\"\"\n html = raw_data.get('html', '')\n if not html:\n return {}\n \n soup = BeautifulSoup(html, 'html.parser')\n " }, { "path": "aggregator/indo_crawler.py", "symbol": "IndoCrawler.transform", "line_start": 259, "line_end": 267, "text": " building = title_h2.get_text(strip=True) if title_h2 else raw_data.get('building', '')\n \n sub_h4 = soup.find('h4', class_='text-muted')\n town = raw_data.get('town', '')\n country = \"France\"\n \n if sub_h4:\n links = sub_h4.find_all('a', class_='link')\n if len(links) >= 1:" }, { "path": "aggregator/indo_crawler.py", "symbol": "IndoCrawler.transform", "line_start": 366, "line_end": 374, "text": " \"raw_str\": str(raw_data[\"builder\"]),\n })\n\n if raw_data.get(\"builder\"):\n data.setdefault(\"extensions\", {}).setdefault(\"indo\", {}).setdefault(\"reference_context\", {})[\"builder\"] = str(raw_data[\"builder\"])\n\n # 4. Images\n #
... \n portfolio_items = soup.find_all('div', class_='portfolio-item')" }, { "path": "aggregator/indo_crawler.py", "symbol": "IndoCrawler.transform", "line_start": 256, "line_end": 264, "text": " #

... Couilly-Pont-aux-Dames ...

\n \n title_h2 = soup.find('h2', class_='mb-0')\n building = title_h2.get_text(strip=True) if title_h2 else raw_data.get('building', '')\n \n sub_h4 = soup.find('h4', class_='text-muted')\n town = raw_data.get('town', '')\n country = \"France\"\n " } ] } }, { "catalogue_id": "S014", "source_key": "iohio", "name": "IOHIO", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Instituto de Órganos Históricos de Oaxaca", "scope_description": "Oaxaca, Mexico", "documentary_unit": "Historical organ profile", "source_description": "IOHIO's bilingual website presents its work on the historic organs of Oaxaca. Aggregator registers it as an organ source; the public site also contains institutional and programme material beyond the organ profiles.", "source_description_references": [ "iohio", "iohio_home", "record_iohio" ], "interpretation": "Its institutional home page supplies context, but only an individual profile can substantiate an instrument's specification or conservation state.", "rights_observation": "No general licence for reproducing the collection was established from the consulted pages.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "iohio", "iohio_home", "record_iohio" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_iohio", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Bilingual instrument profile with labelled technical summary and historical essay", "repeated_information": "Specification is split into left-hand and right-hand stop groups.", "date_and_state_distinctions": "Original construction, later alterations and restoration are distinguished in prose.", "interpretive_limit": "Split registers and cancelled stops need their own attributes; a flat list loses their functional meaning.", "observations": [ "Church and region identify the organ.", "Original maker is unknown; restorers and restoration participants are named.", "Construction, relocation, rebuilding and restoration are described with dates.", "Separate bass and treble stop groups identify cancelled reed stops.", "Keyboard range, short octave and split point are specified.", "Reconstruction of keyboard action and windchest is described.", "Pitch and meantone temperament are stated.", "Bellows construction and wind pressure are stated.", "Case measurements, decoration and keyboard configuration are documented.", "Current use and surviving original components are described.", "Case, pipe-decoration and detail photographs are included.", "The narrative refers to dated historical inventories.", "Comparable instruments are linked." ], "uninspected_or_unfilled_material": [], "narrative": "The La Soledad profile combines a historical account with technical description and photographs. Its specification distinguishes bass and treble registers, describes the short octave and identifies cancelled reed stops. Pitch, temperament, wind pressure and case dimensions are reported in the profile. A transcription should preserve the split-register arrangement and the distinction between surviving, reconstructed and cancelled components." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The adapter reads acquired Squarespace-style content blocks, labelled metadata and detail-page media. The source account concerns a specialist regional organ inventory rather than a uniform global register.", "transformation": "Bold labels and following text supply a metadata dictionary. Builder and construction-date labels feed attribution; pitch has a dedicated field, while case dimensions, keyboard and bellows are retained as notes. Stop blocks are recognised through headings such as bass, treble and left or right hand. The parser therefore recognises a functional split within a manual rather than only conventional named divisions.", "limitations": "A heading-based split should not be mistaken for separate keyboards. Narrative conditions such as cancelled stops, alterations or restoration state require interpretation with the source block. The town can be taken from the page title and the building from reference context; those are different evidential origins." }, "implementation": { "class": "IohioCrawler", "class_line": 17, "path": "aggregator/iohio_crawler.py", "inheritance": [ "IohioCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "IohioCrawler.crawl", "path": "aggregator/iohio_crawler.py", "line_start": 46, "line_end": 143, "inherited": false, "file_sha256": "56703e428800e98ec934346b7307b484e3293e584990412a241dbb3ad5f2a5e8", "self_calls": [ "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 50, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" } ], "html_queries": [ { "line": 78, "expression": "soup.find_all(class_='gallery-grid-item')" }, { "line": 93, "expression": "item.find('a', href=True)" }, { "line": 102, "expression": "item.find(class_='gallery-grid-item-title')" } ], "dictionary_entries": [ { "line": 106, "key": "iohio_id", "value_expression": "iohio_id" }, { "line": 107, "key": "town", "value_expression": "title" }, { "line": 108, "key": "building", "value_expression": "'Church'" }, { "line": 109, "key": "url", "value_expression": "full_url" }, { "line": 110, "key": "country", "value_expression": "'Mexico'" }, { "line": 111, "key": "scraped_at", "value_expression": "datetime.now()" }, { "line": 112, "key": "builder", "value_expression": "''" }, { "line": 113, "key": "year", "value_expression": "''" } ], "subscript_assignments": [] }, { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "line_start": 211, "line_end": 371, "inherited": false, "file_sha256": "56703e428800e98ec934346b7307b484e3293e584990412a241dbb3ad5f2a5e8", "self_calls": [ "_prune_empty" ], "literal_get_calls": [ { "line": 212, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html', '')" }, { "line": 255, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town', '')" }, { "line": 290, "receiver": "data['specifications']", "key": "notes", "expression": "data['specifications'].get('notes', [])" }, { "line": 293, "receiver": "data['specifications']", "key": "notes", "expression": "data['specifications'].get('notes', [])" }, { "line": 296, "receiver": "data['specifications']", "key": "notes", "expression": "data['specifications'].get('notes', [])" }, { "line": 264, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building', '')" }, { "line": 265, "receiver": "metadata", "key": "region", "expression": "metadata.get('region', '')" }, { "line": 276, "receiver": "metadata", "key": "organ construction date", "expression": "metadata.get('organ construction date')" }, { "line": 276, "receiver": "metadata", "key": "date", "expression": "metadata.get('date')" }, { "line": 351, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 351, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 367, "receiver": "raw_data", "key": "iohio_id", "expression": "raw_data.get('iohio_id')" }, { "line": 368, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 356, "receiver": "i", "key": "link", "expression": "i.get('link')" } ], "html_queries": [ { "line": 229, "expression": "soup.find_all(class_='sqs-html-content')" }, { "line": 350, "expression": "soup.find_all('img')" }, { "line": 231, "expression": "block.find_all('strong')" }, { "line": 257, "expression": "soup.find('h1')" }, { "line": 257, "expression": "soup.find(class_='gallery-grid-item-title')" } ], "dictionary_entries": [ { "line": 223, "key": "images", "value_expression": "[]" }, { "line": 223, "key": "audio", "value_expression": "[]" }, { "line": 223, "key": "videos", "value_expression": "[]" }, { "line": 223, "key": "documents", "value_expression": "[]" }, { "line": 262, "key": "country", "value_expression": "'Mexico'" }, { "line": 263, "key": "town", "value_expression": "town" }, { "line": 264, "key": "building", "value_expression": "raw_data.get('building', '')" }, { "line": 265, "key": "region", "value_expression": "metadata.get('region', '')" }, { "line": 266, "key": "status", "value_expression": "'current_location'" }, { "line": 272, "key": "name", "value_expression": "metadata['organ builder']" }, { "line": 273, "key": "activities", "value_expression": "['builder']" }, { "line": 288, "key": "raw_str", "value_expression": "metadata['pitch']" }, { "line": 316, "key": "id", "value_expression": "f\"man-{len(data['specifications']['manuals']) + 1}\"" }, { "line": 316, "key": "name", "value_expression": "div_name" }, { "line": 358, "key": "link", "value_expression": "clean_src" }, { "line": 359, "key": "source", "value_expression": "'IOHIO'" }, { "line": 360, "key": "available", "value_expression": "True" }, { "line": 333, "key": "name", "value_expression": "stop_name" }, { "line": 334, "key": "length", "value_expression": "pitch" }, { "line": 335, "key": "raw_str", "value_expression": "line" }, { "line": 344, "key": "name", "value_expression": "acc_name" }, { "line": 345, "key": "raw_str", "value_expression": "line" } ], "subscript_assignments": [ { "line": 261, "value_line": 261, "key": "locations", "target_expression": "data['locations']", "value_expression": "[{'country': 'Mexico', 'town': town, 'building': raw_data.get('building', ''), 'region': metadata.get('region', ''), 'status': 'current_location'}]" }, { "line": 282, "value_line": 282, "key": "stops", "target_expression": "data['specifications']['stops']", "value_expression": "{}" }, { "line": 283, "value_line": 283, "key": "manuals", "target_expression": "data['specifications']['manuals']", "value_expression": "[]" }, { "line": 284, "value_line": 284, "key": "accessories", "target_expression": "data['specifications']['accessories']", "value_expression": "[]" }, { "line": 223, "value_line": 223, "key": "media", "target_expression": "data['media']", "value_expression": "{'images': [], 'audio': [], 'videos': [], 'documents': []}" }, { "line": 288, "value_line": 288, "key": "pitch", "target_expression": "data['specifications']['pitch']", "value_expression": "[{'raw_str': metadata['pitch']}]" }, { "line": 290, "value_line": 290, "key": "notes", "target_expression": "data['specifications']['notes']", "value_expression": "data['specifications'].get('notes', [])" }, { "line": 293, "value_line": 293, "key": "notes", "target_expression": "data['specifications']['notes']", "value_expression": "data['specifications'].get('notes', [])" }, { "line": 296, "value_line": 296, "key": "notes", "target_expression": "data['specifications']['notes']", "value_expression": "data['specifications'].get('notes', [])" }, { "line": 225, "value_line": 225, "key": "images", "target_expression": "data['media']['images']", "value_expression": "[]" }, { "line": 278, "value_line": 278, "key": "dates", "target_expression": "builder_entry['dates']", "value_expression": "[build_date]" } ] } ], "selected_rule_rows": [ { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "line": 278, "value_line": 278, "key": "dates", "target_expression": "builder_entry['dates']", "value_expression": "[build_date]" }, { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "line": 262, "key": "country", "value_expression": "'Mexico'" }, { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "line": 288, "key": "raw_str", "value_expression": "metadata['pitch']" }, { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "key": "html", "value_expression": "raw_data.get('html', '')", "line": 212, "kind": "input read" }, { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "key": "town", "value_expression": "raw_data.get('town', '')", "line": 255, "kind": "input read" }, { "symbol": "IohioCrawler.transform", "path": "aggregator/iohio_crawler.py", "key": "notes", "value_expression": "data['specifications'].get('notes', [])", "line": 290, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/iohio_crawler.py", "symbol": "IohioCrawler.transform", "line_start": 275, "line_end": 283, "text": " # Check for various date labels\n build_date = metadata.get('organ construction date') or metadata.get('date')\n if build_date:\n builder_entry[\"dates\"] = [build_date]\n data[\"involved\"].append(builder_entry)\n\n # 2. Specifications\n data[\"specifications\"][\"stops\"] = {}\n data[\"specifications\"][\"manuals\"] = []" }, { "path": "aggregator/iohio_crawler.py", "symbol": "IohioCrawler.transform", "line_start": 259, "line_end": 267, "text": " town = title_tag.get_text(strip=True)\n\n data[\"locations\"] = [{\n \"country\": \"Mexico\",\n \"town\": town,\n \"building\": raw_data.get(\"building\", \"\"),\n \"region\": metadata.get('region', ''),\n \"status\": \"current_location\"\n }]" }, { "path": "aggregator/iohio_crawler.py", "symbol": "IohioCrawler.transform", "line_start": 285, "line_end": 293, "text": " \n # Technical fields mapping\n if 'pitch' in metadata:\n data[\"specifications\"][\"pitch\"] = [{\"raw_str\": metadata['pitch']}]\n if 'case measurements' in metadata:\n data[\"specifications\"][\"notes\"] = data[\"specifications\"].get(\"notes\", [])\n data[\"specifications\"][\"notes\"].append(f\"Case measurements: {metadata['case measurements']}\")\n if 'keyboard' in metadata:\n data[\"specifications\"][\"notes\"] = data[\"specifications\"].get(\"notes\", [])" }, { "path": "aggregator/iohio_crawler.py", "symbol": "IohioCrawler.transform", "line_start": 211, "line_end": 217, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n html = raw_data.get('html', '')\n if not html: return {}\n \n soup = BeautifulSoup(html, 'html.parser')\n # Use a fresh template and clean it to remove placeholder objects\n data = clean_template(load_json(\"template_orgbase.json\"))" }, { "path": "aggregator/iohio_crawler.py", "symbol": "IohioCrawler.transform", "line_start": 252, "line_end": 260, "text": " metadata[label] = value\n\n # Town from title or raw_data\n town = raw_data.get('town', '')\n # Check all headers for town\n title_tag = soup.find('h1') or soup.find(class_='gallery-grid-item-title')\n if title_tag:\n town = title_tag.get_text(strip=True)\n" }, { "path": "aggregator/iohio_crawler.py", "symbol": "IohioCrawler.transform", "line_start": 287, "line_end": 295, "text": " if 'pitch' in metadata:\n data[\"specifications\"][\"pitch\"] = [{\"raw_str\": metadata['pitch']}]\n if 'case measurements' in metadata:\n data[\"specifications\"][\"notes\"] = data[\"specifications\"].get(\"notes\", [])\n data[\"specifications\"][\"notes\"].append(f\"Case measurements: {metadata['case measurements']}\")\n if 'keyboard' in metadata:\n data[\"specifications\"][\"notes\"] = data[\"specifications\"].get(\"notes\", [])\n data[\"specifications\"][\"notes\"].append(f\"Keyboard: {metadata['keyboard']}\")\n if 'bellows' in metadata:" } ] } }, { "catalogue_id": "S015", "source_key": "jeux_d_orgues_downloads", "name": "Jeux d'orgues Sample Set Downloads", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Jeux d'orgues", "scope_description": "Historic French organs", "documentary_unit": "Sample-set release and format variant", "source_description": "The project records historic French organs and distributes products in several virtual-instrument and sound-bank formats. Its site lists distinct numbered releases, including Stiehr-Mockers, Silbermann, La Chaise-Dieu and Souvigny.", "source_description_references": [ "jeux_d_orgues_downloads" ], "interpretation": "Product number, organ identity and software format are separate fields. A format named on the project home page should not be assigned to every release without checking that release's page.", "rights_observation": "The home page asserts copyright. Price and download availability do not resolve file-specific licence terms.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "jeux_d_orgues_downloads" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects links matching the producer's numbered sample-set naming pattern. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is download/product entries. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Filename or product numbering identifies a publication unit rather than a physical historical configuration. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "JeuxDOrguesDownloadsCrawler", "class_line": 1211, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "JeuxDOrguesDownloadsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "jeux_d_orgues_downloads", "path": "aggregator/grandorgue_library_crawler.py", "line": 1212, "class": "JeuxDOrguesDownloadsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_downloads", "path": "aggregator/grandorgue_library_crawler.py", "line": 1214, "class": "JeuxDOrguesDownloadsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1216, "line_end": 1263, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1222, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 1220, "expression": "soup.select('a[href*=\"jeux-d-orgues-\"]')" } ], "dictionary_entries": [ { "line": 1231, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Download'}" }, { "line": 1232, "key": "item", "value_expression": "{'title': title, 'producer_label': \"Jeux d'orgues\", 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk', 'GrandOrgue'], 'engine_labels': ['Hauptwerk', 'GrandOrgue'], 'access_model_hint': 'free_or_paid_download', 'availability_status_hint': 'producer_download_page'}" }, { "line": 1241, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'download_url': '', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1248, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': title}" }, { "line": 1252, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1253, "key": "extensions", "value_expression": "{'source_fragment_html': str(parent)}" }, { "line": 1231, "key": "position", "value_expression": "position" }, { "line": 1231, "key": "section_label", "value_expression": "'Download'" }, { "line": 1233, "key": "title", "value_expression": "title" }, { "line": 1234, "key": "producer_label", "value_expression": "\"Jeux d'orgues\"" }, { "line": 1235, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1236, "key": "software_labels", "value_expression": "['Hauptwerk', 'GrandOrgue']" }, { "line": 1237, "key": "engine_labels", "value_expression": "['Hauptwerk', 'GrandOrgue']" }, { "line": 1238, "key": "access_model_hint", "value_expression": "'free_or_paid_download'" }, { "line": 1239, "key": "availability_status_hint", "value_expression": "'producer_download_page'" }, { "line": 1242, "key": "package_title", "value_expression": "title" }, { "line": 1243, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1244, "key": "download_url", "value_expression": "''" }, { "line": 1245, "key": "is_odf_only", "value_expression": "False" }, { "line": 1246, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1249, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 1250, "key": "represented_organ_label", "value_expression": "title" }, { "line": 1253, "key": "source_fragment_html", "value_expression": "str(parent)" }, { "line": 1252, "key": "url", "value_expression": "detail_url" }, { "line": 1252, "key": "link_text", "value_expression": "title" }, { "line": 1252, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1243, "key": "package_kind", "value_expression": "'sample_set'" }, { "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1246, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1245, "key": "is_odf_only", "value_expression": "False" }, { "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1234, "key": "producer_label", "value_expression": "\"Jeux d'orgues\"" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "line_start": 1240, "line_end": 1248, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"download_url\": \"\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {" }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "line_start": 1243, "line_end": 1251, "text": " \"package_kind\": \"sample_set\",\n \"download_url\": \"\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title,\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "line_start": 1242, "line_end": 1250, "text": " \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"download_url\": \"\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "JeuxDOrguesDownloadsCrawler.parse_html", "line_start": 1231, "line_end": 1239, "text": " \"catalog\": {\"position\": position, \"section_label\": \"Download\"},\n \"item\": {\n \"title\": title,\n \"producer_label\": \"Jeux d'orgues\",\n \"description_excerpt\": _excerpt(text),\n \"software_labels\": [\"Hauptwerk\", \"GrandOrgue\"],\n \"engine_labels\": [\"Hauptwerk\", \"GrandOrgue\"],\n \"access_model_hint\": \"free_or_paid_download\",\n \"availability_status_hint\": \"producer_download_page\"," } ] } }, { "catalogue_id": "S016", "source_key": "kirchenorgelforum", "name": "Kirchenorgel-Forum", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Kirchenorgel-Forum", "scope_description": "Pipe, electronic and virtual organs; playing, music and community discussion", "documentary_unit": "Discussion thread and dated post", "source_description": "The forum divides pipe organs from electronic, digital and virtual instruments and provides further sections on scores, interpretation, improvisation and events. The public index was readable without entering an account.", "source_description_references": [ "kirchenorgelforum" ], "interpretation": "Cite a post and its date for a contributor's statement. A thread title alone does not substantiate a technical claim, and a public index does not show that every linked post or attachment is accessible. This package contains no participant prose.", "rights_observation": "No republication permission for post text, attachments or participant data was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "kirchenorgelforum" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Forum pages can arrive as an explicit page list, a serialised multi-page capture or one HTML page. Index context and thread-page evidence are handled separately.", "transformation": "The transform constructs one thread record containing posts and forum context. It fills missing thread metadata from the listing, retains page URLs and captured-page count, and can attach a separately hashed index document. Post, quote, reaction, author, link and media extraction are distinct routines.", "limitations": "One captured thread page does not establish thread completeness. Quoted text and a post author have different attribution. Statements in posts remain attributed claims; the forum is not an instrument inventory or an independent confirmation of a specification." }, "implementation": { "class": "KirchenorgelForumCrawler", "class_line": 33, "path": "aggregator/kirchenorgelforum_crawler.py", "inheritance": [ "KirchenorgelForumCrawler", "BaseCrawler" ], "configuration": { "record_label": { "value": "threads", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 44, "class": "KirchenorgelForumCrawler" }, "transformed_json_id_column": { "value": "thread_id", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 46, "class": "KirchenorgelForumCrawler" } }, "methods": [ { "symbol": "KirchenorgelForumCrawler._extract_images", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 801, "line_end": 836, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 806, "receiver": "image", "key": "src", "expression": "image.get('src')" }, { "line": 820, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 814, "receiver": "image", "key": "alt", "expression": "image.get('alt')" }, { "line": 814, "receiver": "image", "key": "title", "expression": "image.get('title')" } ], "html_queries": [ { "line": 805, "expression": "node.select('img[src]')" }, { "line": 819, "expression": "node.select('a[href]')" } ], "dictionary_entries": [ { "line": 809, "key": "url", "value_expression": "url" }, { "line": 810, "key": "link", "value_expression": "url" }, { "line": 811, "key": "file_name", "value_expression": "os.path.basename(urlparse(url).path) or f'post_{post_id}_{index}.jpg'" }, { "line": 812, "key": "post_id", "value_expression": "post_id" }, { "line": 813, "key": "credit", "value_expression": "author" }, { "line": 814, "key": "caption", "value_expression": "image.get('alt') or image.get('title') or ''" }, { "line": 815, "key": "source_role", "value_expression": "'post_embedded_image'" }, { "line": 816, "key": "status", "value_expression": "'discovered'" }, { "line": 826, "key": "url", "value_expression": "url" }, { "line": 827, "key": "link", "value_expression": "url" }, { "line": 828, "key": "file_name", "value_expression": "os.path.basename(urlparse(url).path) or f'post_{post_id}_{index}.jpg'" }, { "line": 829, "key": "post_id", "value_expression": "post_id" }, { "line": 830, "key": "credit", "value_expression": "author" }, { "line": 831, "key": "caption", "value_expression": "link.get_text(' ', strip=True)" }, { "line": 832, "key": "source_role", "value_expression": "'post_linked_image'" }, { "line": 833, "key": "status", "value_expression": "'discovered'" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._extract_links", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 783, "line_end": 799, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 788, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 787, "expression": "node.select('a[href]')" } ], "dictionary_entries": [ { "line": 793, "key": "url", "value_expression": "urljoin(source_url or self.base_url, href)" }, { "line": 794, "key": "label", "value_expression": "link.get_text(' ', strip=True)" }, { "line": 795, "key": "raw_href", "value_expression": "href" }, { "line": 796, "key": "kind", "value_expression": "'image' if _looks_like_image(href) else 'document' if _looks_like_document(href) else 'internal' if href.startswith('t') or href.startswith('#') else 'external'" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._extract_quotes", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 838, "line_end": 858, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 844, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 842, "expression": "node.select('blockquote')" }, { "line": 843, "expression": "quote.select_one('a[href*=\"#msg\"]')" } ], "dictionary_entries": [ { "line": 851, "key": "type", "value_expression": "'quote'" }, { "line": 852, "key": "post_id", "value_expression": "post_id" }, { "line": 853, "key": "quoted_post_id", "value_expression": "quoted_post_id" }, { "line": 854, "key": "quoted_url", "value_expression": "urljoin(source_url or self.base_url, href) if href else ''" }, { "line": 855, "key": "text", "value_expression": "quote.get_text('\\n', strip=True)" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._extract_reactions", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 861, "line_end": 878, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "startswith" ], "literal_get_calls": [ { "line": 865, "receiver": "user", "key": "class", "expression": "user.get('class')" } ], "html_queries": [ { "line": 863, "expression": "box.select('.postratings .ratinguser')" } ], "dictionary_entries": [ { "line": 872, "key": "type", "value_expression": "'thanks'" }, { "line": 873, "key": "post_id", "value_expression": "post_id" }, { "line": 874, "key": "user_display_name", "value_expression": "user_name" }, { "line": 875, "key": "user_source_id", "value_expression": "user_id" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._media_images_from_posts", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 907, "line_end": 911, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 910, "receiver": "post", "key": "images", "expression": "post.get('images')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._parse_forum_thread_rows", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 571, "line_end": 618, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_forum_id", "_thread_parts" ], "literal_get_calls": [ { "line": 593, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 586, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 582, "expression": "soup.select('a.topiclink, a.threadlink')" }, { "line": 579, "expression": "soup.select_one('h1, .forum-title, .page-title')" }, { "line": 585, "expression": "soup.select('tr.threadrow a[href]')" }, { "line": 575, "expression": "soup.select('#breadcrumbs a, #breadcrumbs strong')" } ], "dictionary_entries": [ { "line": 605, "key": "thread_id", "value_expression": "thread_id" }, { "line": 606, "key": "forum_id", "value_expression": "parsed_forum_id or forum_id" }, { "line": 607, "key": "forum_title", "value_expression": "forum_title" }, { "line": 608, "key": "title", "value_expression": "_dedupe_repeated_label(link.get_text(' ', strip=True))" }, { "line": 609, "key": "author", "value_expression": "author" }, { "line": 610, "key": "reply_count", "value_expression": "_first_int(_select_text(container, '.fposts'))" }, { "line": 611, "key": "view_count", "value_expression": "_views_from_text(text)" }, { "line": 612, "key": "last_post_author", "value_expression": "_last_author_from_container(container) or _last_author_from_text(text)" }, { "line": 613, "key": "last_post_at_raw", "value_expression": "_last_date_from_container(container) or _last_date_from_text(text)" }, { "line": 614, "key": "url", "value_expression": "url" }, { "line": 615, "key": "raw_index_html", "value_expression": "str(container) if container else str(link)" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._parse_post", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 522, "line_end": 569, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_extract_images", "_extract_links", "_extract_quotes", "_extract_reactions", "_parse_user_meta" ], "literal_get_calls": [ { "line": 554, "receiver": "profile_node", "key": "href", "expression": "profile_node.get('href')" }, { "line": 554, "receiver": "profile_node", "key": "href", "expression": "profile_node.get('href')" }, { "line": 555, "receiver": "avatar_node", "key": "src", "expression": "avatar_node.get('src')" }, { "line": 555, "receiver": "avatar_node", "key": "src", "expression": "avatar_node.get('src')" }, { "line": 567, "receiver": "quote", "key": "quoted_post_id", "expression": "quote.get('quoted_post_id')" }, { "line": 529, "receiver": "box", "key": "id", "expression": "box.get('id')" }, { "line": 527, "receiver": "anchor", "key": "name", "expression": "anchor.get('name')" }, { "line": 527, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 554, "receiver": "profile_node", "key": "href", "expression": "profile_node.get('href', '')" } ], "html_queries": [ { "line": 541, "expression": "box.select_one('.xquoteable')" }, { "line": 542, "expression": "box.select_one('.msignature')" }, { "line": 523, "expression": "box.select_one('.boxheader')" }, { "line": 523, "expression": "box.select_one('.card-header')" }, { "line": 524, "expression": "header.select_one('a[name^=\"msg\"]')" }, { "line": 526, "expression": "box.select_one('a[name^=\"msg\"], a[href^=\"#msg\"]')" }, { "line": 531, "expression": "header.select('a strong')" }, { "line": 533, "expression": "header.select_one('.mposttime')" }, { "line": 535, "expression": "header.select_one('.float-right')" }, { "line": 537, "expression": "box.select_one('.messageUser strong')" }, { "line": 537, "expression": "box.select_one('.muser .userlink')" }, { "line": 537, "expression": "box.select_one('.xsuser .userlink')" }, { "line": 539, "expression": "box.select_one('.messageUser a[href]')" }, { "line": 539, "expression": "box.select_one('.muser a[href], .xsuser a[href]')" }, { "line": 540, "expression": "box.select_one('.messageAvatar img')" }, { "line": 540, "expression": "box.select_one('.image_container img, .xsuser img')" } ], "dictionary_entries": [ { "line": 549, "key": "post_id", "value_expression": "post_id" }, { "line": 550, "key": "post_number", "value_expression": "post_number" }, { "line": 551, "key": "url", "value_expression": "f'{source_url}#msg{post_id}' if post_id else source_url" }, { "line": 552, "key": "title", "value_expression": "title" }, { "line": 553, "key": "author_display_name", "value_expression": "author" }, { "line": 554, "key": "author_profile_url", "value_expression": "urljoin(self.base_url, profile_node.get('href')) if profile_node and profile_node.get('href') and (not profile_node.get('href', '').startswith('#')) else ''" }, { "line": 555, "key": "author_avatar_url", "value_expression": "_abs_url(avatar_node.get('src'), self.base_url) if avatar_node and avatar_node.get('src') else ''" }, { "line": 556, "key": "created_at", "value_expression": "_parse_forum_datetime(raw_time)" }, { "line": 557, "key": "created_at_raw", "value_expression": "raw_time" }, { "line": 558, "key": "edited_at_raw", "value_expression": "_edited_text(raw_time)" }, { "line": 559, "key": "html", "value_expression": "str(body_node) if body_node else ''" }, { "line": 560, "key": "text", "value_expression": "body_node.get_text('\\n', strip=True) if body_node else ''" }, { "line": 561, "key": "signature_html", "value_expression": "str(signature_node) if signature_node else ''" }, { "line": 562, "key": "signature_text", "value_expression": "signature_node.get_text('\\n', strip=True) if signature_node else ''" }, { "line": 563, "key": "author_metadata", "value_expression": "user_meta" }, { "line": 564, "key": "links", "value_expression": "links" }, { "line": 565, "key": "images", "value_expression": "images" }, { "line": 566, "key": "quotes", "value_expression": "quotes" }, { "line": 567, "key": "responds_to", "value_expression": "[quote['quoted_post_id'] for quote in quotes if quote.get('quoted_post_id')]" }, { "line": 568, "key": "reactions", "value_expression": "reactions" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._parse_user_meta", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 768, "line_end": 781, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 770, "expression": "box.select('.messageinformation tr')" }, { "line": 775, "expression": "box.select('.userdetails .row')" }, { "line": 771, "expression": "row.find_all('td')" }, { "line": 776, "expression": "row.select_one('.minfo_head')" }, { "line": 777, "expression": "row.select_one('.minfo_cont')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.crawl", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 156, "line_end": 265, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_discover_forums", "_forum_page_url", "_parse_forum_thread_rows", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 161, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 158, "receiver": "kwargs", "key": "max_forum_pages", "expression": "kwargs.get('max_forum_pages')" }, { "line": 158, "receiver": "self.config", "key": "max_forum_pages", "expression": "self.config.get('max_forum_pages', 1)" }, { "line": 159, "receiver": "kwargs", "key": "max_threads", "expression": "kwargs.get('max_threads')" }, { "line": 159, "receiver": "self.config", "key": "max_threads", "expression": "self.config.get('max_threads', 100)" }, { "line": 160, "receiver": "kwargs", "key": "forum_urls", "expression": "kwargs.get('forum_urls')" }, { "line": 160, "receiver": "self.config", "key": "forum_urls", "expression": "self.config.get('forum_urls')" }, { "line": 231, "receiver": "row", "key": "thread_id", "expression": "row.get('thread_id')" }, { "line": 232, "receiver": "row", "key": "forum_id", "expression": "row.get('forum_id')" }, { "line": 233, "receiver": "row", "key": "forum_title", "expression": "row.get('forum_title')" }, { "line": 234, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 235, "receiver": "row", "key": "author", "expression": "row.get('author')" }, { "line": 236, "receiver": "row", "key": "reply_count", "expression": "row.get('reply_count')" }, { "line": 237, "receiver": "row", "key": "view_count", "expression": "row.get('view_count')" }, { "line": 238, "receiver": "row", "key": "last_post_author", "expression": "row.get('last_post_author')" }, { "line": 239, "receiver": "row", "key": "last_post_at_raw", "expression": "row.get('last_post_at_raw')" }, { "line": 240, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 242, "receiver": "row", "key": "raw_index_html", "expression": "row.get('raw_index_html')" }, { "line": 250, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.5)" }, { "line": 190, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.normalized_batch_record", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 432, "line_end": 453, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 450, "receiver": "transformed.get('provenance', {}).get('identity', {})", "key": "crawl_run_id", "expression": "transformed.get('provenance', {}).get('identity', {}).get('crawl_run_id', '')" }, { "line": 450, "receiver": "transformed.get('provenance', {})", "key": "identity", "expression": "transformed.get('provenance', {}).get('identity', {})" }, { "line": 450, "receiver": "transformed", "key": "provenance", "expression": "transformed.get('provenance', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.parse_thread_html", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 455, "line_end": 459, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "parse_thread_pages" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 459, "key": "url", "value_expression": "source_url" }, { "line": 459, "key": "html", "value_expression": "html" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.parse_thread_pages", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 461, "line_end": 520, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_interactions_from_posts", "_links_from_posts", "_media_images_from_posts", "_parse_post", "_participants_from_posts", "_thread_parts" ], "literal_get_calls": [ { "line": 465, "receiver": "pages[0]", "key": "url", "expression": "pages[0].get('url')" }, { "line": 466, "receiver": "pages[0]", "key": "html", "expression": "pages[0].get('html')" }, { "line": 483, "receiver": "page", "key": "url", "expression": "page.get('url')" }, { "line": 488, "receiver": "post", "key": "post_id", "expression": "post.get('post_id')" }, { "line": 484, "receiver": "page", "key": "html", "expression": "page.get('html')" }, { "line": 509, "receiver": "posts[0]", "key": "created_at", "expression": "posts[0].get('created_at', '')" }, { "line": 510, "receiver": "posts[-1]", "key": "created_at", "expression": "posts[-1].get('created_at', '')" }, { "line": 511, "receiver": "posts[0]", "key": "author_display_name", "expression": "posts[0].get('author_display_name', '')" }, { "line": 496, "receiver": "link", "key": "url", "expression": "link.get('url', '')" }, { "line": 518, "receiver": "page", "key": "url", "expression": "page.get('url')" } ], "html_queries": [ { "line": 469, "expression": "first_soup.select('#breadcrumbs a, #breadcrumbs strong')" }, { "line": 485, "expression": "soup.select('#messagewrap > div.box')" }, { "line": 485, "expression": "soup.select('div.message[id^=\"post_\"]')" } ], "dictionary_entries": [ { "line": 498, "key": "forum", "value_expression": "{'forum_id': forum_id, 'category_title': category_title, 'forum_title': forum_title, 'url': urljoin(self.base_url, f'f{forum_id}.html') if forum_id else ''}" }, { "line": 504, "key": "thread", "value_expression": "{'thread_id': thread_id, 'title': thread_title, 'url': source_url or first_url, 'post_count': len(posts), 'created_at': posts[0].get('created_at', '') if posts else '', 'updated_at': posts[-1].get('created_at', '') if posts else '', 'starter': posts[0].get('author_display_name', '') if posts else ''}" }, { "line": 513, "key": "posts", "value_expression": "posts" }, { "line": 514, "key": "participants", "value_expression": "participants" }, { "line": 515, "key": "interactions", "value_expression": "interactions" }, { "line": 516, "key": "links", "value_expression": "links" }, { "line": 517, "key": "media", "value_expression": "{'images': media_images, 'documents': documents}" }, { "line": 518, "key": "source_snapshot", "value_expression": "{'page_urls': [page['url'] for page in pages if page.get('url')]}" }, { "line": 519, "key": "quality", "value_expression": "{'completeness': {'thread_pages_crawled': len(pages)}}" }, { "line": 499, "key": "forum_id", "value_expression": "forum_id" }, { "line": 500, "key": "category_title", "value_expression": "category_title" }, { "line": 501, "key": "forum_title", "value_expression": "forum_title" }, { "line": 502, "key": "url", "value_expression": "urljoin(self.base_url, f'f{forum_id}.html') if forum_id else ''" }, { "line": 505, "key": "thread_id", "value_expression": "thread_id" }, { "line": 506, "key": "title", "value_expression": "thread_title" }, { "line": 507, "key": "url", "value_expression": "source_url or first_url" }, { "line": 508, "key": "post_count", "value_expression": "len(posts)" }, { "line": 509, "key": "created_at", "value_expression": "posts[0].get('created_at', '') if posts else ''" }, { "line": 510, "key": "updated_at", "value_expression": "posts[-1].get('created_at', '') if posts else ''" }, { "line": 511, "key": "starter", "value_expression": "posts[0].get('author_display_name', '') if posts else ''" }, { "line": 517, "key": "images", "value_expression": "media_images" }, { "line": 517, "key": "documents", "value_expression": "documents" }, { "line": 518, "key": "page_urls", "value_expression": "[page['url'] for page in pages if page.get('url')]" }, { "line": 519, "key": "completeness", "value_expression": "{'thread_pages_crawled': len(pages)}" }, { "line": 464, "key": "url", "value_expression": "source_url" }, { "line": 464, "key": "html", "value_expression": "''" }, { "line": 519, "key": "thread_pages_crawled", "value_expression": "len(pages)" } ], "subscript_assignments": [ { "line": 489, "value_line": 489, "key": "page_index", "target_expression": "post['page_index']", "value_expression": "page_index" } ] }, { "symbol": "KirchenorgelForumCrawler.scrape_row_to_transfer_record", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 396, "line_end": 430, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 410, "receiver": "row", "key": "view_count", "expression": "row.get('view_count')" }, { "line": 407, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 408, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 411, "receiver": "row", "key": "last_post_at_raw", "expression": "row.get('last_post_at_raw')" }, { "line": 412, "receiver": "row", "key": "author", "expression": "row.get('author')" }, { "line": 415, "receiver": "row", "key": "forum_id", "expression": "row.get('forum_id')" }, { "line": 416, "receiver": "row", "key": "forum_title", "expression": "row.get('forum_title')" }, { "line": 417, "receiver": "row", "key": "forum_url", "expression": "row.get('forum_url')" }, { "line": 419, "receiver": "row", "key": "scraped_at", "expression": "row.get('scraped_at')" }, { "line": 419, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 425, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 426, "receiver": "row", "key": "raw_html", "expression": "row.get('raw_html')" }, { "line": 426, "receiver": "row", "key": "raw_index_html", "expression": "row.get('raw_index_html')" }, { "line": 409, "receiver": "row", "key": "post_count", "expression": "row.get('post_count')" } ], "html_queries": [], "dictionary_entries": [ { "line": 405, "key": "thread", "value_expression": "{'thread_id': str(row.get(id_column) or ''), 'title': row.get('title') or '', 'url': row.get('url') or '', 'post_count': int(row.get('post_count') or 0), 'view_count': row.get('view_count'), 'updated_at': row.get('last_post_at_raw') or '', 'starter': row.get('author') or ''}" }, { "line": 414, "key": "forum", "value_expression": "{'forum_id': row.get('forum_id') or '', 'forum_title': row.get('forum_title') or '', 'url': row.get('forum_url') or ''}" }, { "line": 419, "key": "source_info", "value_expression": "{'retrieved_at': row.get('scraped_at') or '', 'url': row.get('url') or ''}" }, { "line": 406, "key": "thread_id", "value_expression": "str(row.get(id_column) or '')" }, { "line": 407, "key": "title", "value_expression": "row.get('title') or ''" }, { "line": 408, "key": "url", "value_expression": "row.get('url') or ''" }, { "line": 409, "key": "post_count", "value_expression": "int(row.get('post_count') or 0)" }, { "line": 410, "key": "view_count", "value_expression": "row.get('view_count')" }, { "line": 411, "key": "updated_at", "value_expression": "row.get('last_post_at_raw') or ''" }, { "line": 412, "key": "starter", "value_expression": "row.get('author') or ''" }, { "line": 415, "key": "forum_id", "value_expression": "row.get('forum_id') or ''" }, { "line": 416, "key": "forum_title", "value_expression": "row.get('forum_title') or ''" }, { "line": 417, "key": "url", "value_expression": "row.get('forum_url') or ''" }, { "line": 419, "key": "retrieved_at", "value_expression": "row.get('scraped_at') or ''" }, { "line": 419, "key": "url", "value_expression": "row.get('url') or ''" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 340, "line_end": 394, "inherited": false, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "parse_thread_html", "parse_thread_pages" ], "literal_get_calls": [ { "line": 341, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 341, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 345, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 345, "receiver": "parsed.get('thread', {})", "key": "url", "expression": "parsed.get('thread', {}).get('url')" }, { "line": 358, "receiver": "forum", "key": "forum_id", "expression": "forum.get('forum_id')" }, { "line": 359, "receiver": "forum", "key": "forum_title", "expression": "forum.get('forum_title')" }, { "line": 359, "receiver": "raw_data", "key": "forum_title", "expression": "raw_data.get('forum_title')" }, { "line": 361, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 361, "receiver": "parsed['thread']", "key": "url", "expression": "parsed['thread'].get('url')" }, { "line": 365, "receiver": "raw_data", "key": "raw_index_html", "expression": "raw_data.get('raw_index_html')" }, { "line": 342, "receiver": "raw_data", "key": "pages", "expression": "raw_data.get('pages')" }, { "line": 344, "receiver": "raw_data", "key": "thread_id", "expression": "raw_data.get('thread_id')" }, { "line": 344, "receiver": "parsed.get('thread', {})", "key": "thread_id", "expression": "parsed.get('thread', {}).get('thread_id')" }, { "line": 388, "receiver": "parsed['thread']", "key": "thread_id", "expression": "parsed['thread'].get('thread_id')" }, { "line": 389, "receiver": "parsed['thread']", "key": "url", "expression": "parsed['thread'].get('url')" }, { "line": 392, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 393, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" }, { "line": 343, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 343, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 345, "receiver": "parsed", "key": "thread", "expression": "parsed.get('thread', {})" }, { "line": 358, "receiver": "raw_data", "key": "forum_id", "expression": "raw_data.get('forum_id')" }, { "line": 363, "receiver": "page", "key": "url", "expression": "page.get('url')" }, { "line": 344, "receiver": "parsed", "key": "thread", "expression": "parsed.get('thread', {})" }, { "line": 380, "receiver": "raw_data", "key": "forum_url", "expression": "raw_data.get('forum_url')" }, { "line": 381, "receiver": "raw_data", "key": "scraped_at", "expression": "raw_data.get('scraped_at')" }, { "line": 382, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 374, "receiver": "parsed['thread']", "key": "thread_id", "expression": "parsed['thread'].get('thread_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 347, "key": "title", "value_expression": "('title',)" }, { "line": 348, "key": "view_count", "value_expression": "('view_count',)" }, { "line": 349, "key": "starter", "value_expression": "('author',)" }, { "line": 350, "key": "updated_at", "value_expression": "('last_post_at_raw',)" }, { "line": 372, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, str(parsed['thread'].get('thread_id') or 'unknown'), 'forum_index_html')" }, { "line": 377, "key": "document_kind", "value_expression": "'forum_index_html'" }, { "line": 378, "key": "content_sha256", "value_expression": "index_hash" }, { "line": 379, "key": "content_type", "value_expression": "'text/html'" }, { "line": 380, "key": "url", "value_expression": "raw_data.get('forum_url') or ''" }, { "line": 381, "key": "retrieved_at", "value_expression": "raw_data.get('scraped_at') or ''" }, { "line": 382, "key": "crawl_run_id", "value_expression": "raw_data.get('crawl_run_id') or ''" } ], "subscript_assignments": [ { "line": 344, "value_line": 344, "key": "thread_id", "target_expression": "parsed.setdefault('thread', {})['thread_id']", "value_expression": "str(raw_data.get('thread_id') or parsed.get('thread', {}).get('thread_id') or '')" }, { "line": 345, "value_line": 345, "key": "url", "target_expression": "parsed.setdefault('thread', {})['url']", "value_expression": "raw_data.get('url') or parsed.get('thread', {}).get('url') or ''" }, { "line": 358, "value_line": 358, "key": "forum_id", "target_expression": "forum['forum_id']", "value_expression": "forum.get('forum_id') or str(raw_data.get('forum_id') or '')" }, { "line": 359, "value_line": 359, "key": "forum_title", "target_expression": "forum['forum_title']", "value_expression": "forum.get('forum_title') or raw_data.get('forum_title') or ''" }, { "line": 360, "value_line": 360, "key": "retrieved_at", "target_expression": "parsed.setdefault('source_info', {})['retrieved_at']", "value_expression": "datetime.now().astimezone().isoformat()" }, { "line": 361, "value_line": 361, "key": "url", "target_expression": "parsed['source_info']['url']", "value_expression": "raw_data.get('url') or parsed['thread'].get('url') or ''" }, { "line": 363, "value_line": 363, "key": "page_urls", "target_expression": "parsed.setdefault('source_snapshot', {})['page_urls']", "value_expression": "[page['url'] for page in pages if page.get('url')]" }, { "line": 364, "value_line": 364, "key": "thread_pages_crawled", "target_expression": "parsed.setdefault('quality', {}).setdefault('completeness', {})['thread_pages_crawled']", "value_expression": "len(pages)" }, { "line": 369, "value_line": 369, "key": "raw_index_html", "target_expression": "snapshot['raw_index_html']", "value_expression": "raw_index_html" } ] } ], "selected_rule_rows": [ { "symbol": "KirchenorgelForumCrawler.parse_thread_pages", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 518, "key": "source_snapshot", "value_expression": "{'page_urls': [page['url'] for page in pages if page.get('url')]}" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "html", "value_expression": "raw_data.get('html')", "line": 341, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "raw_html", "value_expression": "raw_data.get('raw_html')", "line": 341, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "url", "value_expression": "raw_data.get('url')", "line": 345, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "forum_id", "value_expression": "forum.get('forum_id')", "line": 358, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "forum_title", "value_expression": "forum.get('forum_title')", "line": 359, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.parse_thread_pages", "line_start": 515, "line_end": 520, "text": " \"interactions\": interactions,\n \"links\": links,\n \"media\": {\"images\": media_images, \"documents\": documents},\n \"source_snapshot\": {\"page_urls\": [page[\"url\"] for page in pages if page.get(\"url\")]},\n \"quality\": {\"completeness\": {\"thread_pages_crawled\": len(pages)}},\n }" }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 340, "line_end": 346, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n html = raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\"\n pages = _normalize_page_list(raw_data.get(\"pages\")) or _deserialize_thread_pages(html)\n parsed = self.parse_thread_pages(pages, raw_data.get(\"url\") or \"\") if pages else self.parse_thread_html(html, raw_data.get(\"url\") or \"\")\n parsed.setdefault(\"thread\", {})[\"thread_id\"] = str(raw_data.get(\"thread_id\") or parsed.get(\"thread\", {}).get(\"thread_id\") or \"\")\n parsed.setdefault(\"thread\", {})[\"url\"] = raw_data.get(\"url\") or parsed.get(\"thread\", {}).get(\"url\") or \"\"\n for target_key, source_keys in {" }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 342, "line_end": 350, "text": " pages = _normalize_page_list(raw_data.get(\"pages\")) or _deserialize_thread_pages(html)\n parsed = self.parse_thread_pages(pages, raw_data.get(\"url\") or \"\") if pages else self.parse_thread_html(html, raw_data.get(\"url\") or \"\")\n parsed.setdefault(\"thread\", {})[\"thread_id\"] = str(raw_data.get(\"thread_id\") or parsed.get(\"thread\", {}).get(\"thread_id\") or \"\")\n parsed.setdefault(\"thread\", {})[\"url\"] = raw_data.get(\"url\") or parsed.get(\"thread\", {}).get(\"url\") or \"\"\n for target_key, source_keys in {\n \"title\": (\"title\",),\n \"view_count\": (\"view_count\",),\n \"starter\": (\"author\",),\n \"updated_at\": (\"last_post_at_raw\",)," }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 355, "line_end": 363, "text": " parsed[\"thread\"][target_key] = raw_data[source_key]\n break\n forum = parsed.setdefault(\"forum\", {})\n forum[\"forum_id\"] = forum.get(\"forum_id\") or str(raw_data.get(\"forum_id\") or \"\")\n forum[\"forum_title\"] = forum.get(\"forum_title\") or raw_data.get(\"forum_title\") or \"\"\n parsed.setdefault(\"source_info\", {})[\"retrieved_at\"] = datetime.now().astimezone().isoformat()\n parsed[\"source_info\"][\"url\"] = raw_data.get(\"url\") or parsed[\"thread\"].get(\"url\") or \"\"\n if pages:\n parsed.setdefault(\"source_snapshot\", {})[\"page_urls\"] = [page[\"url\"] for page in pages if page.get(\"url\")]" }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 356, "line_end": 364, "text": " break\n forum = parsed.setdefault(\"forum\", {})\n forum[\"forum_id\"] = forum.get(\"forum_id\") or str(raw_data.get(\"forum_id\") or \"\")\n forum[\"forum_title\"] = forum.get(\"forum_title\") or raw_data.get(\"forum_title\") or \"\"\n parsed.setdefault(\"source_info\", {})[\"retrieved_at\"] = datetime.now().astimezone().isoformat()\n parsed[\"source_info\"][\"url\"] = raw_data.get(\"url\") or parsed[\"thread\"].get(\"url\") or \"\"\n if pages:\n parsed.setdefault(\"source_snapshot\", {})[\"page_urls\"] = [page[\"url\"] for page in pages if page.get(\"url\")]\n parsed.setdefault(\"quality\", {}).setdefault(\"completeness\", {})[\"thread_pages_crawled\"] = len(pages)" } ] } }, { "catalogue_id": "S017", "source_key": "lars_palo_vpo", "name": "Lars Palo Free GrandOrgue Samplesets", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Lars Palo", "scope_description": "Primarily Swedish organs prepared for GrandOrgue", "documentary_unit": "Sample-set package, variant and patch", "source_description": "The download table distinguishes original and extended stop counts and gives package versions, file sizes and software requirements. Its licensing statement separates older sets under CC BY-SA 2.5 from the 2023 Norrfjärden set under CC BY-SA 4.0.", "source_description_references": [ "lars_palo_vpo" ], "interpretation": "Version and original/extended status are necessary comparison fields. Summing the stop counts of virtual extensions would misdescribe the recorded instrument. A patch belongs to the release it modifies.", "rights_observation": "The page reports Creative Commons licences with differing versions; the downloaded package's own licence remains the item-level reference.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "lars_palo_vpo" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects table rows, then article/download/list nodes as a fallback. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is row-based items plus a specialised detail-enrichment step. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Original and extended versions require separate edition labels; a file link does not establish that both versions reproduce the same specification. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "LarsPaloVpoCrawler", "class_line": 788, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "LarsPaloVpoCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "lars_palo_vpo", "path": "aggregator/grandorgue_library_crawler.py", "line": 789, "class": "LarsPaloVpoCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_downloads", "path": "aggregator/grandorgue_library_crawler.py", "line": 791, "class": "LarsPaloVpoCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "LarsPaloVpoCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 872, "line_end": 878, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 875, "receiver": "package", "key": "checksum_md5", "expression": "package.get('checksum_md5')" }, { "line": 877, "receiver": "package", "key": "grandorgue_unique_id", "expression": "package.get('grandorgue_unique_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 876, "value_line": 876, "key": "checksum_md5", "target_expression": "package['checksum_md5']", "value_expression": "_first_match(text, '(?i)\\\\bMD5\\\\b[:\\\\s]*([a-f0-9]{32})')" }, { "line": 878, "value_line": 878, "key": "grandorgue_unique_id", "target_expression": "package['grandorgue_unique_id']", "value_expression": "_first_match(text, '(?i)(?:GrandOrgue\\\\s*)?(?:unique\\\\s*)?ID[:\\\\s]*([A-Za-z0-9._:-]{6,})')" } ] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "LarsPaloVpoCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 793, "line_end": 870, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [], "html_queries": [ { "line": 795, "expression": "soup.select('table tr')" }, { "line": 797, "expression": "soup.select('article, .download, li')" }, { "line": 800, "expression": "node.select('td, th')" } ], "dictionary_entries": [ { "line": 830, "key": "catalog", "value_expression": "{'position': position}" }, { "line": 831, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Lars Palo', 'description_excerpt': _excerpt(text), 'version_label': version, 'license_label': _first_match(text, '(?i)(Creative Commons[^.]+|GPL[^.]+|freeware)') or 'free', 'access_model_hint': 'free_download'}" }, { "line": 839, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'package_format': _package_format(download_url), 'download_url': download_url, 'file_name': _file_name(download_url), 'file_size_label': file_size_label, 'file_size_bytes': _file_size_bytes(file_size_label), 'checksum_md5': md5, 'grandorgue_unique_id': unique_id, 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 852, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': title, 'manual_count': manual_count, 'pedal_present': 'pedal' in text.lower(), 'stop_count': stop_count}" }, { "line": 859, "key": "links", "value_expression": "links" }, { "line": 860, "key": "extensions", "value_expression": "{'source_fragment_html': str(node)}" }, { "line": 830, "key": "position", "value_expression": "position" }, { "line": 832, "key": "title", "value_expression": "title" }, { "line": 833, "key": "producer_label", "value_expression": "'Lars Palo'" }, { "line": 834, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 835, "key": "version_label", "value_expression": "version" }, { "line": 836, "key": "license_label", "value_expression": "_first_match(text, '(?i)(Creative Commons[^.]+|GPL[^.]+|freeware)') or 'free'" }, { "line": 837, "key": "access_model_hint", "value_expression": "'free_download'" }, { "line": 840, "key": "package_title", "value_expression": "title" }, { "line": 841, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 842, "key": "package_format", "value_expression": "_package_format(download_url)" }, { "line": 843, "key": "download_url", "value_expression": "download_url" }, { "line": 844, "key": "file_name", "value_expression": "_file_name(download_url)" }, { "line": 845, "key": "file_size_label", "value_expression": "file_size_label" }, { "line": 846, "key": "file_size_bytes", "value_expression": "_file_size_bytes(file_size_label)" }, { "line": 847, "key": "checksum_md5", "value_expression": "md5" }, { "line": 848, "key": "grandorgue_unique_id", "value_expression": "unique_id" }, { "line": 849, "key": "is_odf_only", "value_expression": "False" }, { "line": 850, "key": "requires_sample_set", "value_expression": "False" }, { "line": 853, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 854, "key": "represented_organ_label", "value_expression": "title" }, { "line": 855, "key": "manual_count", "value_expression": "manual_count" }, { "line": 856, "key": "pedal_present", "value_expression": "'pedal' in text.lower()" }, { "line": 857, "key": "stop_count", "value_expression": "stop_count" }, { "line": 860, "key": "source_fragment_html", "value_expression": "str(node)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "LarsPaloVpoCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 857, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "LarsPaloVpoCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 855, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "LarsPaloVpoCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 841, "key": "package_kind", "value_expression": "'sample_set'" }, { "symbol": "LarsPaloVpoCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 850, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "LarsPaloVpoCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 849, "key": "is_odf_only", "value_expression": "False" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LarsPaloVpoCrawler.parse_html", "line_start": 854, "line_end": 862, "text": " \"represented_organ_label\": title,\n \"manual_count\": manual_count,\n \"pedal_present\": \"pedal\" in text.lower(),\n \"stop_count\": stop_count,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)},\n },\n primary_id=slugify(title)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LarsPaloVpoCrawler.parse_html", "line_start": 852, "line_end": 860, "text": " \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title,\n \"manual_count\": manual_count,\n \"pedal_present\": \"pedal\" in text.lower(),\n \"stop_count\": stop_count,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LarsPaloVpoCrawler.parse_html", "line_start": 838, "line_end": 846, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"package_format\": _package_format(download_url),\n \"download_url\": download_url,\n \"file_name\": _file_name(download_url),\n \"file_size_label\": file_size_label,\n \"file_size_bytes\": _file_size_bytes(file_size_label)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LarsPaloVpoCrawler.parse_html", "line_start": 847, "line_end": 855, "text": " \"checksum_md5\": md5,\n \"grandorgue_unique_id\": unique_id,\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title,\n \"manual_count\": manual_count," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LarsPaloVpoCrawler.parse_html", "line_start": 846, "line_end": 854, "text": " \"file_size_bytes\": _file_size_bytes(file_size_label),\n \"checksum_md5\": md5,\n \"grandorgue_unique_id\": unique_id,\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title," } ] } }, { "catalogue_id": "S018", "source_key": "lavender_audio_hereford", "name": "Lavender Audio Hereford Hauptwerk Sample Sets", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Lavender Audio", "scope_description": "One physical organ represented by multiple virtual editions", "documentary_unit": "Hereford sample-set edition", "source_description": "The page distinguishes an evaluation set, 23-stop and 46-stop editions, and the complete 67-stop instrument. It also identifies an XL extension and gives April 2009 and February 2010 as sampling dates.", "source_description_references": [ "lavender_audio_hereford" ], "interpretation": "These products form a particularly clear example of one organ yielding several catalogue records. Sampling date, selected ranks and extended keyboard compass must remain separate from the historical organ's specification.", "rights_observation": "The page retains copyright in samples, images and definition files and specifies licensed software use. The evaluation edition is not an open-data release.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "lavender_audio_hereford" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects page links selected within the Hereford product context. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is edition and file references. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Reduced and extended editions are separate virtual configurations associated with one physical instrument; their stop totals should not be pooled. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "LavenderAudioHerefordCrawler", "class_line": 1704, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "LavenderAudioHerefordCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "lavender_audio_hereford", "path": "aggregator/grandorgue_library_crawler.py", "line": 1705, "class": "LavenderAudioHerefordCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 1707, "class": "LavenderAudioHerefordCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "LavenderAudioHerefordCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1709, "line_end": 1786, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1718, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 1713, "expression": "soup.select('a[href]')" } ], "dictionary_entries": [ { "line": 1729, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Hauptwerk Sample Sets'}" }, { "line": 1730, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Lavender Audio', 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'evaluation' if is_eval else 'commercial', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1739, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'evaluation_sample_set' if is_eval else 'sample_set', 'is_odf_only': False, 'requires_sample_set': False, 'is_demo': is_eval}" }, { "line": 1746, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': 'Hereford Cathedral', 'manual_count': 4, 'pedal_present': True, 'stop_count': stops}" }, { "line": 1753, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1754, "key": "extensions", "value_expression": "{'source_fragment_html': str(link)}" }, { "line": 1772, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Hauptwerk Sample Sets'}" }, { "line": 1773, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Lavender Audio', 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk']}" }, { "line": 1774, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1775, "key": "links", "value_expression": "[{'url': source_url, 'link_text': 'Hereford Cathedral Sample Sets', 'is_product_homepage_candidate': True}]" }, { "line": 1776, "key": "extensions", "value_expression": "{'source_fragment_html': title}" }, { "line": 1729, "key": "position", "value_expression": "position" }, { "line": 1729, "key": "section_label", "value_expression": "'Hauptwerk Sample Sets'" }, { "line": 1731, "key": "title", "value_expression": "title" }, { "line": 1732, "key": "producer_label", "value_expression": "'Lavender Audio'" }, { "line": 1733, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1734, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1735, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1736, "key": "access_model_hint", "value_expression": "'evaluation' if is_eval else 'commercial'" }, { "line": 1737, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1740, "key": "package_title", "value_expression": "title" }, { "line": 1741, "key": "package_kind", "value_expression": "'evaluation_sample_set' if is_eval else 'sample_set'" }, { "line": 1742, "key": "is_odf_only", "value_expression": "False" }, { "line": 1743, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1744, "key": "is_demo", "value_expression": "is_eval" }, { "line": 1747, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 1748, "key": "represented_organ_label", "value_expression": "'Hereford Cathedral'" }, { "line": 1749, "key": "manual_count", "value_expression": "4" }, { "line": 1750, "key": "pedal_present", "value_expression": "True" }, { "line": 1751, "key": "stop_count", "value_expression": "stops" }, { "line": 1754, "key": "source_fragment_html", "value_expression": "str(link)" }, { "line": 1772, "key": "position", "value_expression": "position" }, { "line": 1772, "key": "section_label", "value_expression": "'Hauptwerk Sample Sets'" }, { "line": 1773, "key": "title", "value_expression": "title" }, { "line": 1773, "key": "producer_label", "value_expression": "'Lavender Audio'" }, { "line": 1773, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1773, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1774, "key": "package_title", "value_expression": "title" }, { "line": 1774, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1774, "key": "is_odf_only", "value_expression": "False" }, { "line": 1774, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1776, "key": "source_fragment_html", "value_expression": "title" }, { "line": 1753, "key": "url", "value_expression": "detail_url" }, { "line": 1753, "key": "link_text", "value_expression": "title" }, { "line": 1753, "key": "is_product_homepage_candidate", "value_expression": "True" }, { "line": 1775, "key": "url", "value_expression": "source_url" }, { "line": 1775, "key": "link_text", "value_expression": "'Hereford Cathedral Sample Sets'" }, { "line": 1775, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "LavenderAudioHerefordCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1751, "key": "stop_count", "value_expression": "stops" }, { "symbol": "LavenderAudioHerefordCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1749, "key": "manual_count", "value_expression": "4" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "LavenderAudioHerefordCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1741, "key": "package_kind", "value_expression": "'evaluation_sample_set' if is_eval else 'sample_set'" }, { "symbol": "LavenderAudioHerefordCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1743, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "LavenderAudioHerefordCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1742, "key": "is_odf_only", "value_expression": "False" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LavenderAudioHerefordCrawler.parse_html", "line_start": 1748, "line_end": 1756, "text": " \"represented_organ_label\": \"Hereford Cathedral\",\n \"manual_count\": 4,\n \"pedal_present\": True,\n \"stop_count\": stops,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)},\n },\n primary_id=slugify(title)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LavenderAudioHerefordCrawler.parse_html", "line_start": 1746, "line_end": 1754, "text": " \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": \"Hereford Cathedral\",\n \"manual_count\": 4,\n \"pedal_present\": True,\n \"stop_count\": stops,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LavenderAudioHerefordCrawler.parse_html", "line_start": 1738, "line_end": 1746, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"evaluation_sample_set\" if is_eval else \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": is_eval,\n },\n \"organ_relation\": {" }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LavenderAudioHerefordCrawler.parse_html", "line_start": 1740, "line_end": 1748, "text": " \"package_title\": title,\n \"package_kind\": \"evaluation_sample_set\" if is_eval else \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": is_eval,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": \"Hereford Cathedral\"," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "LavenderAudioHerefordCrawler.parse_html", "line_start": 1739, "line_end": 1747, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"evaluation_sample_set\" if is_eval else \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": is_eval,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\"," } ] } }, { "catalogue_id": "S019", "source_key": "milan_digital_audio_instruments", "name": "Milan Digital Audio Instruments", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Milan Digital Audio", "scope_description": "Selected historical and modern instruments for Hauptwerk", "documentary_unit": "Commercial virtual-organ product", "source_description": "The indexed official catalogue identifies products based on organs at St Eucaire, Salisbury, Bovenkerk, Notre Dame de Metz, Mt Carmel and St Peter and Paul. Direct requests to the site returned HTTP 403 in this review.", "source_description_references": [ "indexed_milan", "milan_digital_audio_instruments" ], "interpretation": "The indexed listing supports identification of the catalogue, not current prices, download availability or release compatibility. Those fields remain unverified. Record the product separately from the physical instrument.", "rights_observation": "Product licence terms were not accessible in the direct check.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "indexed_milan", "milan_digital_audio_instruments" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects article elements and their product links. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is instrument product articles. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A product page can outlast the release it describes; format, version and availability need their own dated evidence. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "MilanDigitalAudioInstrumentsCrawler", "class_line": 1469, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "MilanDigitalAudioInstrumentsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "milan_digital_audio_instruments", "path": "aggregator/grandorgue_library_crawler.py", "line": 1470, "class": "MilanDigitalAudioInstrumentsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 1472, "class": "MilanDigitalAudioInstrumentsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1474, "line_end": 1519, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1483, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" }, { "line": 1483, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" } ], "html_queries": [ { "line": 1476, "expression": "soup.select('article')" }, { "line": 1482, "expression": "node.select_one('a[href*=\"milandigitalaudio.com/\"]')" } ], "dictionary_entries": [ { "line": 1488, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Instruments'}" }, { "line": 1489, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Milan Digital Audio', 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'commercial', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1498, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1504, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': title}" }, { "line": 1508, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1509, "key": "extensions", "value_expression": "{'source_fragment_html': str(node)}" }, { "line": 1488, "key": "position", "value_expression": "position" }, { "line": 1488, "key": "section_label", "value_expression": "'Instruments'" }, { "line": 1490, "key": "title", "value_expression": "title" }, { "line": 1491, "key": "producer_label", "value_expression": "'Milan Digital Audio'" }, { "line": 1492, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1493, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1494, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1495, "key": "access_model_hint", "value_expression": "'commercial'" }, { "line": 1496, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1499, "key": "package_title", "value_expression": "title" }, { "line": 1500, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1501, "key": "is_odf_only", "value_expression": "False" }, { "line": 1502, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1505, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 1506, "key": "represented_organ_label", "value_expression": "title" }, { "line": 1509, "key": "source_fragment_html", "value_expression": "str(node)" }, { "line": 1508, "key": "url", "value_expression": "detail_url" }, { "line": 1508, "key": "link_text", "value_expression": "title" }, { "line": 1508, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1500, "key": "package_kind", "value_expression": "'sample_set'" }, { "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1502, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1501, "key": "is_odf_only", "value_expression": "False" }, { "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1491, "key": "producer_label", "value_expression": "'Milan Digital Audio'" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "line_start": 1497, "line_end": 1505, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\"," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "line_start": 1499, "line_end": 1507, "text": " \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title,\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "line_start": 1498, "line_end": 1506, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MilanDigitalAudioInstrumentsCrawler.parse_html", "line_start": 1488, "line_end": 1496, "text": " \"catalog\": {\"position\": position, \"section_label\": \"Instruments\"},\n \"item\": {\n \"title\": title,\n \"producer_label\": \"Milan Digital Audio\",\n \"description_excerpt\": _excerpt(text),\n \"software_labels\": [\"Hauptwerk\"],\n \"engine_labels\": [\"Hauptwerk\"],\n \"access_model_hint\": \"commercial\",\n \"availability_status_hint\": \"producer_catalog_entry\"," } ] } }, { "catalogue_id": "S020", "source_key": "mps_orgelseite_filebase", "name": "MPS Orgelseite Filebase", "selection_role": "registered_source", "parent_source_key": null, "attribution": "MPS Orgelseite contributors", "scope_description": "Community resources for GrandOrgue and Hauptwerk", "documentary_unit": "Filebase entry, including ODFs, composites and other resources", "source_description": "The current filebase mixes definition files, composite organs, simulations, tools and musical material. Entries expose dates, revisions and download counters, with separate GrandOrgue and Hauptwerk categories.", "source_description_references": [ "mps_orgelseite_filebase" ], "interpretation": "A filebase category is not a homogeneous sample-set collection. Classify each entry before joining it to organs or products. Public metadata and downloadable bytes have different access conditions; no attachments were retrieved here.", "rights_observation": "Permissions depend on the contribution and on any underlying samples it requires.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "mps_orgelseite_filebase" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects filebase links, file IDs, article/file-entry containers and entry-title links. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is file entries and their detail/download context. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A filebase item may be an update, definition or supporting file; item identity is not an organ identity. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "MpsOrgelseiteFilebaseCrawler", "class_line": 576, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "MpsOrgelseiteFilebaseCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "mps_orgelseite_filebase", "path": "aggregator/grandorgue_library_crawler.py", "line": 577, "class": "MpsOrgelseiteFilebaseCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "filebase", "path": "aggregator/grandorgue_library_crawler.py", "line": 579, "class": "MpsOrgelseiteFilebaseCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 581, "line_end": 646, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 585, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 600, "receiver": "card", "key": "data-file-id", "expression": "card.get('data-file-id')" }, { "line": 598, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" }, { "line": 598, "receiver": "title_link", "key": "href", "expression": "title_link.get('href')" } ], "html_queries": [ { "line": 584, "expression": "soup.select('a[href*=\"index.php?file/\"]')" }, { "line": 591, "expression": "soup.select('[data-file-id], article, .filebaseFile, .filebaseEntry')" }, { "line": 594, "expression": "node.select_one('a[href*=\"filebase\"], a[href*=\"file\"], h3 a, h2 a, .filebaseFileTitle a')" } ], "dictionary_entries": [ { "line": 607, "key": "url", "value_expression": "detail_url" }, { "line": 607, "key": "link_text", "value_expression": "'Detail page'" }, { "line": 607, "key": "is_product_homepage_candidate", "value_expression": "True" }, { "line": 617, "key": "catalog", "value_expression": "{'category_label': category, 'category_url': source_url, 'position': position}" }, { "line": 618, "key": "item", "value_expression": "{'title': title, 'provider_label': _select_text(card, '.username, .author, .userLink'), 'description_excerpt': _excerpt(text), 'download_count': download_count, 'access_model_hint': 'registration_required' if requires_login else 'public_or_unknown', 'availability_status_hint': 'available_with_registration' if requires_login else 'available_or_unknown'}" }, { "line": 626, "key": "package", "value_expression": "{'package_title': title, 'package_kind': package_kind, 'download_url': _first_download_url(links), 'is_odf_only': is_odf, 'requires_sample_set': is_odf or is_patch, 'is_patch': is_patch, 'is_composite': is_composite}" }, { "line": 635, "key": "links", "value_expression": "links" }, { "line": 636, "key": "extensions", "value_expression": "{'source_fragment_html': str(card)}" }, { "line": 617, "key": "category_label", "value_expression": "category" }, { "line": 617, "key": "category_url", "value_expression": "source_url" }, { "line": 617, "key": "position", "value_expression": "position" }, { "line": 619, "key": "title", "value_expression": "title" }, { "line": 620, "key": "provider_label", "value_expression": "_select_text(card, '.username, .author, .userLink')" }, { "line": 621, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 622, "key": "download_count", "value_expression": "download_count" }, { "line": 623, "key": "access_model_hint", "value_expression": "'registration_required' if requires_login else 'public_or_unknown'" }, { "line": 624, "key": "availability_status_hint", "value_expression": "'available_with_registration' if requires_login else 'available_or_unknown'" }, { "line": 627, "key": "package_title", "value_expression": "title" }, { "line": 628, "key": "package_kind", "value_expression": "package_kind" }, { "line": 629, "key": "download_url", "value_expression": "_first_download_url(links)" }, { "line": 630, "key": "is_odf_only", "value_expression": "is_odf" }, { "line": 631, "key": "requires_sample_set", "value_expression": "is_odf or is_patch" }, { "line": 632, "key": "is_patch", "value_expression": "is_patch" }, { "line": 633, "key": "is_composite", "value_expression": "is_composite" }, { "line": 636, "key": "source_fragment_html", "value_expression": "str(card)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 628, "key": "package_kind", "value_expression": "package_kind" }, { "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 631, "key": "requires_sample_set", "value_expression": "is_odf or is_patch" }, { "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 630, "key": "is_odf_only", "value_expression": "is_odf" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "line_start": 625, "line_end": 633, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": package_kind,\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf,\n \"requires_sample_set\": is_odf or is_patch,\n \"is_patch\": is_patch,\n \"is_composite\": is_composite," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "line_start": 628, "line_end": 636, "text": " \"package_kind\": package_kind,\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf,\n \"requires_sample_set\": is_odf or is_patch,\n \"is_patch\": is_patch,\n \"is_composite\": is_composite,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(card)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "MpsOrgelseiteFilebaseCrawler.parse_html", "line_start": 627, "line_end": 635, "text": " \"package_title\": title,\n \"package_kind\": package_kind,\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf,\n \"requires_sample_set\": is_odf or is_patch,\n \"is_patch\": is_patch,\n \"is_composite\": is_composite,\n },\n \"links\": links," } ] } }, { "catalogue_id": "S021", "source_key": "musiqueorguequebec", "name": "Musique et musiciens - Orgues", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Musique et musiciens / Music and musicians", "scope_description": "Organ-related pages; wider musical content on the parent site", "documentary_unit": "Organ profile within a broader music website", "source_description": "The configured root identifies a bilingual music website rather than an organ-only register. Aggregator's organ adapter selects profiles from this larger resource. The root-page response by itself gives no reliable organ-record count.", "source_description_references": [ "musiqueorguequebec", "selection_scope", "record_musiqueorguequebec" ], "interpretation": "Catalogue references should point to the organ profile, not the general home page. The site's name should not be used to infer that every documented instrument is in Québec.", "rights_observation": "No general licence for the selected organ profiles was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "musiqueorguequebec", "selection_scope", "record_musiqueorguequebec" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_musiqueorguequebec", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Bilingual church-and-organ essay with comparative specification tables", "repeated_information": "Original and current stops appear side by side within each division.", "date_and_state_distinctions": "Building history and organ history coexist. Original and later specifications are explicitly differentiated.", "interpretive_limit": "Parallel original/current columns must remain aligned; church events must not become organ construction events.", "observations": [ "Church, municipality and province identify the page.", "Original builder/opus and a later builder are named.", "Dated building and organ narratives are separate sections.", "Original and current specifications are compared by division.", "Manual and pedal ranges and note counts are provided.", "Electro-pneumatic action is stated.", "Couplers, adjustable combinations and expression controls are listed.", "The page includes an instrument photograph and enlarged-image link.", "A dedicated bibliography closes the page.", "Internal language/specification anchors and image links are present." ], "uninspected_or_unfilled_material": [], "narrative": "The Bic, Sainte-Cécile page separates building history from organ history and presents original and later specifications in parallel columns. The columns preserve relationships between successive states of each division. Manual and pedal ranges, action and playing aids accompany the stoplist. Reading either the building chronology or the two specification columns as one undifferentiated organ record would change the account." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Index discovery leads to bilingual or French organ articles. The adapter parses page text and stop tables with source-specific routines.", "transformation": "Builder history is parsed from narrative patterns. Manual, stop and rank totals are sought through explicit textual expressions; stop count falls back to the number of extracted entries. Stop tables are preserved as raw specification text and division groups, while action, images and location have separate extraction paths.", "limitations": "Parallel original and later specifications need state-specific reading. A grouped stop dictionary does not automatically preserve the semantics of parallel historical columns. Declared totals and sums of parsed entries should be reported with their respective origins." }, "implementation": { "class": "MusiqueOrgueQuebecCrawler", "class_line": 2797, "path": "aggregator/static_html_crawler.py", "inheritance": [ "MusiqueOrgueQuebecCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "musiqueorguequebec", "path": "aggregator/static_html_crawler.py", "line": 2798, "class": "MusiqueOrgueQuebecCrawler" }, "id_column": { "value": "moq_id", "path": "aggregator/static_html_crawler.py", "line": 2799, "class": "MusiqueOrgueQuebecCrawler" }, "default_country": { "value": "Canada", "path": "aggregator/static_html_crawler.py", "line": 2802, "class": "MusiqueOrgueQuebecCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "http://www.musiqueorguequebec.ca/orgues/orgueque3.html", "http://www.musiqueorguequebec.ca/orgues/orguecan3.html" ], "path": "aggregator/static_html_crawler.py", "line": 2803, "class": "MusiqueOrgueQuebecCrawler" }, "detail_url_patterns": { "value": [ "/orgues/quebec/", "/orgues/canada/" ], "path": "aggregator/static_html_crawler.py", "line": 2807, "class": "MusiqueOrgueQuebecCrawler" }, "exclude_url_patterns": { "value": [ ".jpg", ".gif", ".png", ".pdf", "repertoiredesorgues.qc.ca" ], "path": "aggregator/static_html_crawler.py", "line": 2808, "class": "MusiqueOrgueQuebecCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "http://www.musiqueorguequebec.ca", "path": "aggregator/static_html_crawler.py", "line": 2801, "class": "MusiqueOrgueQuebecCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "MusiqueOrgueQuebecCrawler._builder_history", "path": "aggregator/static_html_crawler.py", "line_start": 2907, "line_end": 2920, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 2919, "key": "builder_name", "value_expression": "builder" }, { "line": 2919, "key": "year", "value_expression": "year" }, { "line": 2919, "key": "activity", "value_expression": "'builder'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MusiqueOrgueQuebecCrawler._extract_stop_tables", "path": "aggregator/static_html_crawler.py", "line_start": 2930, "line_end": 2958, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 2933, "expression": "soup.find_all('table')" }, { "line": 2934, "expression": "table.find_all('tr')" }, { "line": 2940, "expression": "row.find_all(['td', 'th'], recursive=False)" } ], "dictionary_entries": [ { "line": 2951, "key": "name", "value_expression": "name" } ], "subscript_assignments": [ { "line": 2953, "value_line": 2953, "key": "length", "target_expression": "item['length']", "value_expression": "length" } ] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MusiqueOrgueQuebecCrawler._location_from_title", "path": "aggregator/static_html_crawler.py", "line_start": 2897, "line_end": 2905, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 2898, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 2899, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MusiqueOrgueQuebecCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 2810, "line_end": 2844, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_identifier_from_url", "_is_detail_url", "_is_region_heading", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 2814, "expression": "soup.find_all('tr')" }, { "line": 2815, "expression": "row.find_all(['td', 'th'], recursive=False)" }, { "line": 2822, "expression": "row.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 2840, "key": "index_url", "value_expression": "page_url" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 2849, "line_end": 2895, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_builder_history", "_extract_images", "_extract_stop_tables", "_extract_title", "_first_regex_int", "_location_from_title", "_traction" ], "literal_get_calls": [ { "line": 2850, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 2886, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 2892, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 2885, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 2891, "receiver": "raw_data", "key": "moq_id", "expression": "raw_data.get('moq_id')" }, { "line": 2876, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 2884, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2862, "key": "manual_count", "value_expression": "manual_count" }, { "line": 2863, "key": "stop_count", "value_expression": "stop_count or sum((len(stops) for stops in stops_by_division.values()))" }, { "line": 2864, "key": "rank_count", "value_expression": "rank_count" }, { "line": 2865, "key": "traction_key", "value_expression": "self._traction(text)" }, { "line": 2866, "key": "raw_specification", "value_expression": "raw_spec" }, { "line": 2867, "key": "stops", "value_expression": "stops_by_division" }, { "line": 2868, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 2872, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': self.default_country, 'region': raw_data.get('region') or '', 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 2882, "key": "builder_history", "value_expression": "builder_history" }, { "line": 2883, "key": "specifications", "value_expression": "specifications" }, { "line": 2884, "key": "media", "value_expression": "{'images': self._extract_images(soup, raw_data.get('url') or self.base_url)}" }, { "line": 2885, "key": "links", "value_expression": "[{'link': raw_data.get('url')}]" }, { "line": 2886, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', '')}" }, { "line": 2884, "key": "images", "value_expression": "self._extract_images(soup, raw_data.get('url') or self.base_url)" }, { "line": 2886, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 2874, "key": "status", "value_expression": "'current_location'" }, { "line": 2875, "key": "country", "value_expression": "self.default_country" }, { "line": 2876, "key": "region", "value_expression": "raw_data.get('region') or ''" }, { "line": 2877, "key": "town", "value_expression": "town" }, { "line": 2878, "key": "building", "value_expression": "building" }, { "line": 2879, "key": "raw_str", "value_expression": "title" }, { "line": 2885, "key": "link", "value_expression": "raw_data.get('url')" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2863, "key": "stop_count", "value_expression": "stop_count or sum((len(stops) for stops in stops_by_division.values()))" }, { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2862, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2864, "key": "rank_count", "value_expression": "rank_count" }, { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2868, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2866, "key": "raw_specification", "value_expression": "raw_spec" }, { "symbol": "MusiqueOrgueQuebecCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2875, "key": "country", "value_expression": "self.default_country" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "MusiqueOrgueQuebecCrawler.transform", "line_start": 2860, "line_end": 2868, "text": "\n specifications = {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count or sum(len(stops) for stops in stops_by_division.values()),\n \"rank_count\": rank_count,\n \"traction_key\": self._traction(text),\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division," }, { "path": "aggregator/static_html_crawler.py", "symbol": "MusiqueOrgueQuebecCrawler.transform", "line_start": 2859, "line_end": 2867, "text": " town, building = self._location_from_title(raw_data, title)\n\n specifications = {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count or sum(len(stops) for stops in stops_by_division.values()),\n \"rank_count\": rank_count,\n \"traction_key\": self._traction(text),\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division," }, { "path": "aggregator/static_html_crawler.py", "symbol": "MusiqueOrgueQuebecCrawler.transform", "line_start": 2861, "line_end": 2869, "text": " specifications = {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count or sum(len(stops) for stops in stops_by_division.values()),\n \"rank_count\": rank_count,\n \"traction_key\": self._traction(text),\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "MusiqueOrgueQuebecCrawler.transform", "line_start": 2865, "line_end": 2873, "text": " \"traction_key\": self._traction(text),\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n }\n\n data = {\n \"locations\": [\n {" }, { "path": "aggregator/static_html_crawler.py", "symbol": "MusiqueOrgueQuebecCrawler.transform", "line_start": 2863, "line_end": 2871, "text": " \"stop_count\": stop_count or sum(len(stops) for stops in stops_by_division.values()),\n \"rank_count\": rank_count,\n \"traction_key\": self._traction(text),\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n }\n\n data = {" }, { "path": "aggregator/static_html_crawler.py", "symbol": "MusiqueOrgueQuebecCrawler.transform", "line_start": 2872, "line_end": 2880, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": self.default_country,\n \"region\": raw_data.get(\"region\") or \"\",\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" } ] } }, { "catalogue_id": "S022", "source_key": "musixplora", "name": "musiXplora", "selection_role": "registered_source", "parent_source_key": null, "attribution": "musiXplora / Universität Leipzig", "scope_description": "Persons, institutions, objects or works, places, events, concepts and titles", "documentary_unit": "Typed entity and relation", "source_description": "The public interface separates seven lexical domains. Its person search exposes several external authority identifiers, while the project description presents musiXplora as an expanding organological and musicological reference resource.", "source_description_references": [ "browser_musixplora", "selection_scope" ], "interpretation": "Entity and relation identifiers can support comparison across collections. Observations of the public interface do not establish the coverage or date of a separately obtained data extract.", "rights_observation": "Open access is stated by the project; a licence for redistributing the local dump was not established by the public pages.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "browser_musixplora", "selection_scope" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler facade loads local dump tables through MusiXploraGraphBuilder, builds source-native entities and caches references. It does not acquire the public website in this path. Dump provenance and the public portal description therefore represent different evidence.", "transformation": "The normalizer receives an entity and an optional manifest. Graph construction, entity normalization and an optional pipe-organ projection are separate layers. Relations, entity family/type and source IDs belong to the dump model; organ relevance is a selection assessment rather than an entity identity.", "limitations": "A public portal page cannot independently verify every field in a local dump. Entity, relationship and organ-projection counts have different denominators. The organ projection must not erase non-organ entities or attribute all graph relations to physical instruments." }, "implementation": { "class": "MusiXploraCrawler", "class_line": 19, "path": "aggregator/musixplora/crawler.py", "inheritance": [ "MusiXploraCrawler", "BaseCrawler" ], "configuration": { "record_label": { "value": "entities", "path": "aggregator/musixplora/crawler.py", "line": 23, "class": "MusiXploraCrawler" } }, "methods": [ { "symbol": "MusiXploraCrawler._ensure_indexes", "path": "aggregator/musixplora/crawler.py", "line_start": 265, "line_end": 283, "inherited": false, "file_sha256": "4a04e0e7e33fba162e6920cc4aa1a2be4fbde24738b40f5982fe646a6a3afeed", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MusiXploraCrawler.crawl", "path": "aggregator/musixplora/crawler.py", "line_start": 51, "line_end": 111, "inherited": false, "file_sha256": "4a04e0e7e33fba162e6920cc4aa1a2be4fbde24738b40f5982fe646a6a3afeed", "self_calls": [ "_cache_row", "_mark_scrape_status", "_source_dir", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 54, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 55, "receiver": "kwargs", "key": "limit", "expression": "kwargs.get('limit')" }, { "line": 55, "receiver": "self.config", "key": "index_limit", "expression": "self.config.get('index_limit')" }, { "line": 53, "receiver": "kwargs", "key": "source_dir", "expression": "kwargs.get('source_dir')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "MusiXploraCrawler.preflight_crawl", "path": "aggregator/musixplora/crawler.py", "line_start": 227, "line_end": 235, "inherited": false, "file_sha256": "4a04e0e7e33fba162e6920cc4aa1a2be4fbde24738b40f5982fe646a6a3afeed", "self_calls": [ "_source_dir" ], "literal_get_calls": [ { "line": 228, "receiver": "task_kwargs or {}", "key": "source_dir", "expression": "(task_kwargs or {}).get('source_dir')" } ], "html_queries": [], "dictionary_entries": [ { "line": 235, "key": "ok", "value_expression": "True" }, { "line": 235, "key": "message", "value_expression": "'musiXplora dump directory is available.'" }, { "line": 235, "key": "source_dir", "value_expression": "str(source_dir)" }, { "line": 234, "key": "ok", "value_expression": "False" }, { "line": 234, "key": "message", "value_expression": "f'musiXplora dump directory does not exist: {source_dir}'" }, { "line": 232, "key": "ok", "value_expression": "False" }, { "line": 232, "key": "message", "value_expression": "str(exc)" } ], "subscript_assignments": [] }, { "symbol": "MusiXploraCrawler.transform", "path": "aggregator/musixplora/crawler.py", "line_start": 113, "line_end": 116, "inherited": false, "file_sha256": "4a04e0e7e33fba162e6920cc4aa1a2be4fbde24738b40f5982fe646a6a3afeed", "self_calls": [], "literal_get_calls": [ { "line": 114, "receiver": "raw_data", "key": "entity", "expression": "raw_data.get('entity')" }, { "line": 115, "receiver": "raw_data", "key": "manifest", "expression": "raw_data.get('manifest')" }, { "line": 114, "receiver": "raw_data", "key": "entity", "expression": "raw_data.get('entity')" }, { "line": 115, "receiver": "raw_data", "key": "manifest", "expression": "raw_data.get('manifest')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "MusiXploraCrawler.transform", "path": "aggregator/musixplora/crawler.py", "key": "entity", "value_expression": "raw_data.get('entity')", "line": 114, "kind": "input read" }, { "symbol": "MusiXploraCrawler.transform", "path": "aggregator/musixplora/crawler.py", "key": "manifest", "value_expression": "raw_data.get('manifest')", "line": 115, "kind": "input read" }, { "symbol": "MusiXploraCrawler.crawl", "path": "aggregator/musixplora/crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 54, "kind": "input read" }, { "symbol": "MusiXploraCrawler.crawl", "path": "aggregator/musixplora/crawler.py", "key": "limit", "value_expression": "kwargs.get('limit')", "line": 55, "kind": "input read" }, { "symbol": "MusiXploraCrawler.crawl", "path": "aggregator/musixplora/crawler.py", "key": "index_limit", "value_expression": "self.config.get('index_limit')", "line": 55, "kind": "input read" }, { "symbol": "MusiXploraCrawler.crawl", "path": "aggregator/musixplora/crawler.py", "key": "source_dir", "value_expression": "kwargs.get('source_dir')", "line": 53, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/musixplora/crawler.py", "symbol": "MusiXploraCrawler.transform", "line_start": 113, "line_end": 116, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n entity = raw_data.get(\"entity\") if isinstance(raw_data.get(\"entity\"), dict) else raw_data\n manifest = raw_data.get(\"manifest\") if isinstance(raw_data.get(\"manifest\"), list) else []\n return normalize_musixplora_entity_record(entity, manifest=manifest)" }, { "path": "aggregator/musixplora/crawler.py", "symbol": "MusiXploraCrawler.crawl", "line_start": 51, "line_end": 59, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n \"\"\"Load dump rows, build source-native entity references, and cache them.\"\"\"\n source_dir = self._source_dir(kwargs.get(\"source_dir\") or target)\n task_id = kwargs.get(\"task_id\")\n limit = kwargs.get(\"limit\") or self.config.get(\"index_limit\")\n if limit not in (None, \"\", \"All\"):\n limit = int(limit)\n else:\n limit = None" }, { "path": "aggregator/musixplora/crawler.py", "symbol": "MusiXploraCrawler.crawl", "line_start": 52, "line_end": 60, "text": " \"\"\"Load dump rows, build source-native entity references, and cache them.\"\"\"\n source_dir = self._source_dir(kwargs.get(\"source_dir\") or target)\n task_id = kwargs.get(\"task_id\")\n limit = kwargs.get(\"limit\") or self.config.get(\"index_limit\")\n if limit not in (None, \"\", \"All\"):\n limit = int(limit)\n else:\n limit = None\n" }, { "path": "aggregator/musixplora/crawler.py", "symbol": "MusiXploraCrawler.crawl", "line_start": 51, "line_end": 58, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n \"\"\"Load dump rows, build source-native entity references, and cache them.\"\"\"\n source_dir = self._source_dir(kwargs.get(\"source_dir\") or target)\n task_id = kwargs.get(\"task_id\")\n limit = kwargs.get(\"limit\") or self.config.get(\"index_limit\")\n if limit not in (None, \"\", \"All\"):\n limit = int(limit)\n else:" } ] } }, { "catalogue_id": "S023", "source_key": "nomine", "name": "Nomine", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Norddeutsche Orgelmusikkultur in Niedersachsen und Europa e.V. (NOMINE)", "scope_description": "Historical organs between the Ems and Elbe", "documentary_unit": "Regional organ portrait", "source_description": "NOMINE organises portraits by East Frisia, Oldenburg, Stade and Lüneburg and links the instruments to recordings and video portraits. Its introduction describes a regional heritage of 300 historic organs.", "source_description_references": [ "nomine", "record_nomine" ], "interpretation": "The stated regional total is not a verified count of downloadable records. The combination of portraits and recordings supports contextual study, while a representative inventory would require a separate account of selection and omissions.", "rights_observation": "The site marks its content as protected; no open licence for text, photographs or recordings was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "nomine", "record_nomine" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_nomine", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Regional organ portrait with explanatory prose, stop table and dated work history", "repeated_information": "Division columns, technical rows and a separate construction/restoration chronology.", "date_and_state_distinctions": "Current instrument, surviving earlier case and retained ranks are explicitly distinguished.", "interpretive_limit": "Count surviving components separately from the date of the present instrument.", "observations": [ "Church, place and navigation address are supplied.", "Builders and their interventions are named.", "A chronological work history accompanies the present specification.", "Stops retain division, pitch and mixture-rank information.", "Manual/pedal compass and division labels are given.", "Historical windchest and action changes are discussed.", "Surviving case and coupling arrangements are described.", "Retention of earlier case and registers is explicit.", "Dated bibliography and literature citations are supplied.", "A documentary stand date is attached to the account." ], "uninspected_or_unfilled_material": [], "narrative": "The Abbehausen portrait distinguishes the present instrument from its earlier case and retained registers. A division-based stop table, key compasses, work chronology and bibliography accompany the account. The construction date of the present organ and the dates of surviving components therefore answer different historical questions." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The acquired organ page is interpreted through its title, disposition heading, construction/restoration-history heading and uploaded images.", "transformation": "The title is split into town and building. The first table reached from the disposition heading is passed to a specialised parser. History-table rows provide dates and participant wording. The image filter selects upload paths while excluding certain logo, icon and thumbnail patterns. Manual, pedal, stop and other specification groups are assembled separately.", "limitations": "History cells are retained as participant wording rather than independently resolved organisations or people. A next-table search can cross a section boundary if the expected table is absent. The disposition and history need their own state interpretation; the title split and default country are implementation choices." }, "implementation": { "class": "NomineCrawler", "class_line": 17, "path": "aggregator/nomine_crawler.py", "inheritance": [ "NomineCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "NomineCrawler._parse_disposition_table", "path": "aggregator/nomine_crawler.py", "line_start": 340, "line_end": 504, "inherited": false, "file_sha256": "db87891d26884e5bf316dfc8051046ee0f37a2268dfbf2706d5226e8e10953e0", "self_calls": [], "literal_get_calls": [ { "line": 377, "receiver": "cell.attrs", "key": "colspan", "expression": "cell.attrs.get('colspan', implied_colspan)" } ], "html_queries": [ { "line": 344, "expression": "table.find_all('tr')" }, { "line": 362, "expression": "row.find_all(['th', 'td'])" }, { "line": 410, "expression": "row.find_all('td')" }, { "line": 357, "expression": "row.find_all('td')" } ], "dictionary_entries": [ { "line": 501, "key": "name", "value_expression": "stop_name" }, { "line": 502, "key": "length", "value_expression": "pitch" }, { "line": 503, "key": "raw_str", "value_expression": "f'{stop_name} {pitch}'.strip()" }, { "line": 386, "key": "id", "value_expression": "f\"pedal-{len(data['specifications']['pedals']) + 1}\"" }, { "line": 386, "key": "name", "value_expression": "manual_name" }, { "line": 389, "key": "id", "value_expression": "f\"manual-{len(data['specifications']['manuals']) + 1}\"" }, { "line": 389, "key": "name", "value_expression": "manual_name" } ], "subscript_assignments": [] }, { "symbol": "NomineCrawler.crawl", "path": "aggregator/nomine_crawler.py", "line_start": 48, "line_end": 165, "inherited": false, "file_sha256": "db87891d26884e5bf316dfc8051046ee0f37a2268dfbf2706d5226e8e10953e0", "self_calls": [ "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 52, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" } ], "html_queries": [ { "line": 79, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 126, "key": "nomine_id", "value_expression": "item['nomine_id']" }, { "line": 127, "key": "town", "value_expression": "town" }, { "line": 128, "key": "country", "value_expression": "'Germany'" }, { "line": 129, "key": "building", "value_expression": "building" }, { "line": 130, "key": "url", "value_expression": "item['url']" }, { "line": 131, "key": "scraped_at", "value_expression": "datetime.now()" }, { "line": 132, "key": "builder", "value_expression": "''" }, { "line": 133, "key": "year", "value_expression": "''" }, { "line": 90, "key": "nomine_id", "value_expression": "slug" }, { "line": 91, "key": "url", "value_expression": "href" }, { "line": 92, "key": "title", "value_expression": "link.get_text(strip=True)" } ], "subscript_assignments": [] }, { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "line_start": 249, "line_end": 338, "inherited": false, "file_sha256": "db87891d26884e5bf316dfc8051046ee0f37a2268dfbf2706d5226e8e10953e0", "self_calls": [ "_parse_disposition_table" ], "literal_get_calls": [ { "line": 250, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html', '')" }, { "line": 321, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 334, "receiver": "raw_data", "key": "nomine_id", "expression": "raw_data.get('nomine_id')" }, { "line": 335, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" } ], "html_queries": [ { "line": 266, "expression": "soup.find('h1')" }, { "line": 287, "expression": "soup.find(lambda tag: tag.name in ['h2', 'h3'] and 'Disposition' in tag.get_text())" }, { "line": 296, "expression": "soup.find(lambda tag: tag.name in ['h2', 'h3'] and 'Bau-/Restaurierungsgeschichte' in tag.get_text())" }, { "line": 319, "expression": "soup.find_all('img')" }, { "line": 304, "expression": "hist_table.find_all('tr')" }, { "line": 306, "expression": "row.find_all('td')" } ], "dictionary_entries": [ { "line": 279, "key": "country", "value_expression": "'Germany'" }, { "line": 280, "key": "town", "value_expression": "town" }, { "line": 281, "key": "building", "value_expression": "building" }, { "line": 282, "key": "status", "value_expression": "'current_location'" }, { "line": 326, "key": "link", "value_expression": "src" }, { "line": 327, "key": "source", "value_expression": "'Nomine'" }, { "line": 328, "key": "available", "value_expression": "True" }, { "line": 312, "key": "name", "value_expression": "builder_text" }, { "line": 313, "key": "dates", "value_expression": "[year]" }, { "line": 314, "key": "raw_str", "value_expression": "f'{year}: {builder_text}'" } ], "subscript_assignments": [ { "line": 261, "value_line": 261, "key": "stops", "target_expression": "data['specifications']['stops']", "value_expression": "{}" }, { "line": 262, "value_line": 262, "key": "pedals", "target_expression": "data['specifications']['pedals']", "value_expression": "[]" }, { "line": 263, "value_line": 263, "key": "manuals", "target_expression": "data['specifications']['manuals']", "value_expression": "[]" }, { "line": 278, "value_line": 278, "key": "locations", "target_expression": "data['locations']", "value_expression": "[{'country': 'Germany', 'town': town, 'building': building, 'status': 'current_location'}]" } ] } ], "selected_rule_rows": [ { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "line": 313, "key": "dates", "value_expression": "[year]" }, { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "line": 279, "key": "country", "value_expression": "'Germany'" }, { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "line": 314, "key": "raw_str", "value_expression": "f'{year}: {builder_text}'" }, { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "key": "html", "value_expression": "raw_data.get('html', '')", "line": 250, "kind": "input read" }, { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "key": "src", "value_expression": "img.get('src')", "line": 321, "kind": "input read" }, { "symbol": "NomineCrawler.transform", "path": "aggregator/nomine_crawler.py", "key": "nomine_id", "value_expression": "raw_data.get('nomine_id')", "line": 334, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/nomine_crawler.py", "symbol": "NomineCrawler.transform", "line_start": 310, "line_end": 318, "text": " if year and builder_text and (year.strip() or builder_text.strip()):\n data[\"involved\"].append({\n \"name\": builder_text,\n \"dates\": [year],\n \"raw_str\": f\"{year}: {builder_text}\"\n })\n\n # 4. Images\n # Nomine uses galleries or single images." }, { "path": "aggregator/nomine_crawler.py", "symbol": "NomineCrawler.transform", "line_start": 276, "line_end": 284, "text": " town = title_text\n \n data[\"locations\"] = [{\n \"country\": \"Germany\", # Nomine focuses on Northern Germany/Lower Saxony\n \"town\": town,\n \"building\": building,\n \"status\": \"current_location\"\n }]\n" }, { "path": "aggregator/nomine_crawler.py", "symbol": "NomineCrawler.transform", "line_start": 311, "line_end": 319, "text": " data[\"involved\"].append({\n \"name\": builder_text,\n \"dates\": [year],\n \"raw_str\": f\"{year}: {builder_text}\"\n })\n\n # 4. Images\n # Nomine uses galleries or single images.\n images = soup.find_all('img')" }, { "path": "aggregator/nomine_crawler.py", "symbol": "NomineCrawler.transform", "line_start": 249, "line_end": 255, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n html = raw_data.get('html', '')\n if not html:\n return {}\n \n soup = BeautifulSoup(html, 'html.parser')\n data = copy.deepcopy(self.template)" }, { "path": "aggregator/nomine_crawler.py", "symbol": "NomineCrawler.transform", "line_start": 318, "line_end": 326, "text": " # Nomine uses galleries or single images.\n images = soup.find_all('img')\n for img in images:\n src = img.get('src')\n if src and ('wp-content/uploads' in src) and not ('logo' in src) and not ('icon' in src):\n # Filter out small thumbnails if possible, but Nomine seems to link to full size usually\n if '-150x150' not in src: \n data[\"media\"][\"images\"].append({\n \"link\": src," }, { "path": "aggregator/nomine_crawler.py", "symbol": "NomineCrawler.transform", "line_start": 331, "line_end": 338, "text": " return normalize_transfer_record(\n data,\n source_key=\"nomine\",\n primary_id=raw_data.get(\"nomine_id\"),\n source_url=raw_data.get(\"url\"),\n raw_content=html,\n content_type=\"text/html\",\n )" } ] } }, { "catalogue_id": "S024", "source_key": "npor", "name": "NPOR", "selection_role": "registered_source", "parent_source_key": null, "attribution": "British Institute of Organ Studies (BIOS)", "scope_description": "National Pipe Organ Register and linked British organ resources", "documentary_unit": "Source survey linked to an organ", "source_description": "BIOS provides the register, whose content is managed under an agreement with the Royal College of Organists. The current interface advertises more than 37,000 organs and links to the Historic Organ Sound Archive, British Organ Archive and Directory of British Organ Builders.", "source_description_references": [ "browser_npor", "rendered_record_npor" ], "interpretation": "A survey identifier identifies a documentary record. Repeated surveys or revised descriptions must not be counted automatically as different physical organs. The linked archives are additional resources, not separate registered Aggregator keys in this inventory.", "rights_observation": "Copyright is displayed. No general open licence for surveys, archive images or sound recordings was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "browser_npor", "rendered_record_npor" ], "observation_date": "2026-09-10", "selected_document": { "reference": "rendered_record_npor", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Survey record with separate history, builder and archive links", "repeated_information": "Departments contain action, compass and stop rows. Survey updates and photographs are separate repeatable sections.", "date_and_state_distinctions": "Survey date, last update, building history, work dates and photograph dates are distinct.", "interpretive_limit": "Survey identifiers distinguish accounts. Related surveys may describe earlier organs or previous locations.", "observations": [ "Church address, postcode and building type are supplied.", "National Grid reference links to a neighbourhood search.", "Named builder linked to a builder record.", "Relocation/rebuilding account links to earlier and related surveys.", "Department tables provide stop names, pitches and explanatory notes.", "Department-specific key compasses and note counts are populated.", "Key and stop action are stated separately for departments.", "Equal temperament is specified.", "Electric blowing is recorded.", "Separate case and console descriptions include control details.", "The survey marks the organ playable.", "Dated, credited case and console photographs are available.", "A British Organ Archive reference and source codes are supplied.", "Related survey, builder and history identifiers are linked.", "Survey date, input date, source code and dated updates are separate." ], "uninspected_or_unfilled_material": [], "narrative": "Survey D00002 has department-level stop tables, separate action and compass information, case and console descriptions, photographs and dated updates. It links to builder and related survey/history records and supplies a British Organ Archive reference. Its location and playable state belong to the dated survey. Earlier or related surveys cannot be counted as additional physical organs without examining their relationship." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Discovery supports historic-county searches and explicit survey identifiers. Search results contain survey references; the detail request is keyed by the survey identifier. Search-reference context is retained alongside detail data, and disagreement in location fields is recorded.", "transformation": "The transform separates building and place identifiers from the survey identifier. Builders are iterated as source work assertions, with their ordinal, wording, date and attribution retained. Linked history remains documentary context; editorial updates remain record updates. Archive citations have their own extraction routine. Specification, photographic and bibliographic arrays are handled separately. A fixed normalization epoch is used by the inspected release policy, while acquisition dates belong to the surrounding evidence.", "limitations": "Survey date, editorial update date and organ-work date have different meanings. A survey is not automatically one instrument over its entire lifetime. The code can establish the handling of builder assertions and source references, but cannot validate the historical accuracy of their wording. Detail/search conflicts should be examined rather than resolved by silently accepting the first value." }, "implementation": { "class": "NPORCrawler", "class_line": 16, "path": "aggregator/npor_crawler.py", "inheritance": [ "NPORCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "NPORCrawler._crawl_all", "path": "aggregator/npor_crawler.py", "line_start": 204, "line_end": 254, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_perform_search", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler._crawl_single_county", "path": "aggregator/npor_crawler.py", "line_start": 256, "line_end": 267, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_perform_search", "_update_scrape_status" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler._crawl_survey_ids", "path": "aggregator/npor_crawler.py", "line_start": 134, "line_end": 171, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_fetch_survey", "_normalize_survey_ids", "_reference_from_survey", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler._event_type", "path": "aggregator/npor_crawler.py", "line_start": 686, "line_end": 700, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_descriptions", "path": "aggregator/npor_crawler.py", "line_start": 736, "line_end": 769, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_as_dict_list", "_clean_source_text" ], "literal_get_calls": [ { "line": 739, "receiver": "organ", "key": "information", "expression": "organ.get('information', [])" }, { "line": 738, "receiver": "survey", "key": "organ", "expression": "survey.get('organ')" }, { "line": 744, "receiver": "survey", "key": "history", "expression": "survey.get('history')" }, { "line": 745, "receiver": "history", "key": "title", "expression": "history.get('title')" }, { "line": 756, "receiver": "survey", "key": "updates", "expression": "survey.get('updates')" }, { "line": 738, "receiver": "survey", "key": "organ", "expression": "survey.get('organ')" }, { "line": 744, "receiver": "survey", "key": "history", "expression": "survey.get('history')" }, { "line": 757, "receiver": "update", "key": "text", "expression": "update.get('text')" }, { "line": 751, "receiver": "history", "key": "link", "expression": "history.get('link')" }, { "line": 764, "receiver": "update", "key": "date", "expression": "update.get('date')" }, { "line": 765, "receiver": "update", "key": "name", "expression": "update.get('name')" } ], "html_queries": [], "dictionary_entries": [ { "line": 749, "key": "type", "value_expression": "'linked_history_context'" }, { "line": 750, "key": "text", "value_expression": "history_title" }, { "line": 751, "key": "source_native_link", "value_expression": "self._clean_source_text(history.get('link'))" }, { "line": 752, "key": "event_inferred", "value_expression": "False" }, { "line": 762, "key": "type", "value_expression": "'source_record_update'" }, { "line": 763, "key": "text", "value_expression": "text" }, { "line": 764, "key": "date", "value_expression": "self._clean_source_text(update.get('date'))" }, { "line": 765, "key": "contributor", "value_expression": "self._clean_source_text(update.get('name'))" }, { "line": 766, "key": "event_inferred", "value_expression": "False" }, { "line": 742, "key": "type", "value_expression": "'source_information'" }, { "line": 742, "key": "text", "value_expression": "text" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_events", "path": "aggregator/npor_crawler.py", "line_start": 702, "line_end": 734, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_as_dict_list", "_clean_source_text", "_event_type", "_work_text" ], "literal_get_calls": [ { "line": 705, "receiver": "survey", "key": "builders", "expression": "survey.get('builders')" }, { "line": 706, "receiver": "entry", "key": "builder", "expression": "entry.get('builder')" }, { "line": 708, "receiver": "entry", "key": "date", "expression": "entry.get('date')" }, { "line": 706, "receiver": "entry", "key": "builder", "expression": "entry.get('builder')" }, { "line": 707, "receiver": "nested", "key": "name", "expression": "nested.get('name')" }, { "line": 707, "receiver": "entry", "key": "name", "expression": "entry.get('name')" }, { "line": 724, "receiver": "entry", "key": "work_type", "expression": "entry.get('work_type')" }, { "line": 725, "receiver": "entry", "key": "text", "expression": "entry.get('text')" } ], "html_queries": [], "dictionary_entries": [ { "line": 715, "key": "id", "value_expression": "f'builder-assertion-{index + 1}'" }, { "line": 716, "key": "type", "value_expression": "event_type" }, { "line": 717, "key": "dates", "value_expression": "[date] if date else []" }, { "line": 718, "key": "activities", "value_expression": "activities" }, { "line": 719, "key": "involved", "value_expression": "[{'name': name, 'activities': activities}] if name else []" }, { "line": 720, "key": "description", "value_expression": "work or None" }, { "line": 721, "key": "source_native", "value_expression": "{'section': 'builders', 'ordinal': index, 'work_type': self._clean_source_text(entry.get('work_type')), 'text': self._clean_source_text(entry.get('text'))}" }, { "line": 727, "key": "derivation", "value_expression": "{'method': 'deterministic_source_field_projection', 'policy': 'npor-builder-assertion-events-v1', 'inferred': False}" }, { "line": 722, "key": "section", "value_expression": "'builders'" }, { "line": 723, "key": "ordinal", "value_expression": "index" }, { "line": 724, "key": "work_type", "value_expression": "self._clean_source_text(entry.get('work_type'))" }, { "line": 725, "key": "text", "value_expression": "self._clean_source_text(entry.get('text'))" }, { "line": 728, "key": "method", "value_expression": "'deterministic_source_field_projection'" }, { "line": 729, "key": "policy", "value_expression": "'npor-builder-assertion-events-v1'" }, { "line": 730, "key": "inferred", "value_expression": "False" }, { "line": 719, "key": "name", "value_expression": "name" }, { "line": 719, "key": "activities", "value_expression": "activities" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_involved", "path": "aggregator/npor_crawler.py", "line_start": 641, "line_end": 655, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_as_dict_list", "_work_text" ], "literal_get_calls": [ { "line": 643, "receiver": "survey", "key": "builders", "expression": "survey.get('builders')" }, { "line": 646, "receiver": "b_entry['builder']", "key": "name", "expression": "b_entry['builder'].get('name', '')" }, { "line": 648, "receiver": "b_entry", "key": "name", "expression": "b_entry.get('name', '')" }, { "line": 652, "receiver": "b_entry", "key": "date", "expression": "b_entry.get('date', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 651, "key": "name", "value_expression": "name" }, { "line": 652, "key": "year", "value_expression": "b_entry.get('date', '')" }, { "line": 653, "key": "type_of_work", "value_expression": "self._work_text(b_entry)" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_literature", "path": "aggregator/npor_crawler.py", "line_start": 896, "line_end": 938, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_as_dict_list", "_clean_source_text" ], "literal_get_calls": [ { "line": 901, "receiver": "survey", "key": "archives", "expression": "survey.get('archives')" }, { "line": 902, "receiver": "archive", "key": "sources", "expression": "archive.get('sources')" }, { "line": 910, "receiver": "source", "key": "type", "expression": "source.get('type')" }, { "line": 921, "receiver": "native", "key": "author", "expression": "native.get('author', '')" }, { "line": 923, "receiver": "native", "key": "date", "expression": "native.get('date', '')" }, { "line": 924, "receiver": "native", "key": "page", "expression": "native.get('page', '')" }, { "line": 922, "receiver": "native", "key": "title", "expression": "native.get('title')" }, { "line": 922, "receiver": "native", "key": "document", "expression": "native.get('document', '')" }, { "line": 928, "receiver": "archive", "key": "boa-ref", "expression": "archive.get('boa-ref')" }, { "line": 929, "receiver": "archive", "key": "building", "expression": "archive.get('building')" }, { "line": 930, "receiver": "archive", "key": "town", "expression": "archive.get('town')" } ], "html_queries": [], "dictionary_entries": [ { "line": 919, "key": "type", "value_expression": "f'npor_archive_{source_type}'" }, { "line": 920, "key": "raw_str", "value_expression": "'; '.join(citation_parts)" }, { "line": 921, "key": "author", "value_expression": "native.get('author', '')" }, { "line": 922, "key": "title", "value_expression": "native.get('title') or native.get('document', '')" }, { "line": 923, "key": "date", "value_expression": "native.get('date', '')" }, { "line": 924, "key": "pages", "value_expression": "native.get('page', '')" }, { "line": 925, "key": "source_native", "value_expression": "native" }, { "line": 926, "key": "archive_context", "value_expression": "{'ordinal': archive_index, 'boa_ref': self._clean_source_text(archive.get('boa-ref')), 'building': self._clean_source_text(archive.get('building')), 'town': self._clean_source_text(archive.get('town'))}" }, { "line": 932, "key": "derivation", "value_expression": "{'method': 'deterministic_source_field_projection', 'policy': 'npor-archive-sources-v1'}" }, { "line": 927, "key": "ordinal", "value_expression": "archive_index" }, { "line": 928, "key": "boa_ref", "value_expression": "self._clean_source_text(archive.get('boa-ref'))" }, { "line": 929, "key": "building", "value_expression": "self._clean_source_text(archive.get('building'))" }, { "line": 930, "key": "town", "value_expression": "self._clean_source_text(archive.get('town'))" }, { "line": 933, "key": "method", "value_expression": "'deterministic_source_field_projection'" }, { "line": 934, "key": "policy", "value_expression": "'npor-archive-sources-v1'" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_locations", "path": "aggregator/npor_crawler.py", "line_start": 624, "line_end": 639, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [ { "line": 626, "receiver": "survey", "key": "building", "expression": "survey.get('building', {})" }, { "line": 631, "receiver": "building_info", "key": "town", "expression": "building_info.get('town', '')" }, { "line": 632, "receiver": "building_info", "key": "historic_county", "expression": "building_info.get('historic_county', '')" }, { "line": 633, "receiver": "building_info", "key": "county", "expression": "building_info.get('county', '')" }, { "line": 634, "receiver": "building_info", "key": "name", "expression": "building_info.get('name', '')" }, { "line": 635, "receiver": "building_info", "key": "address", "expression": "building_info.get('address', '')" }, { "line": 636, "receiver": "building_info", "key": "postcode", "expression": "building_info.get('postcode', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 631, "key": "town", "value_expression": "building_info.get('town', '')" }, { "line": 632, "key": "historic_county", "value_expression": "building_info.get('historic_county', '')" }, { "line": 633, "key": "county", "value_expression": "building_info.get('county', '')" }, { "line": 634, "key": "building", "value_expression": "building_info.get('name', '')" }, { "line": 635, "key": "address", "value_expression": "building_info.get('address', '')" }, { "line": 636, "key": "postcode", "value_expression": "building_info.get('postcode', '')" }, { "line": 637, "key": "country", "value_expression": "'United Kingdom'" }, { "line": 638, "key": "region", "value_expression": "''" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_main_builder", "path": "aggregator/npor_crawler.py", "line_start": 350, "line_end": 357, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [ { "line": 351, "receiver": "survey", "key": "builders", "expression": "survey.get('builders', [])" }, { "line": 356, "receiver": "builder", "key": "name", "expression": "builder.get('name', '')" }, { "line": 354, "receiver": "builder", "key": "builder", "expression": "builder.get('builder')" }, { "line": 355, "receiver": "builder['builder']", "key": "name", "expression": "builder['builder'].get('name', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_media", "path": "aggregator/npor_crawler.py", "line_start": 855, "line_end": 894, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [ { "line": 858, "receiver": "survey", "key": "photographs", "expression": "survey.get('photographs', [])" }, { "line": 873, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 867, "receiver": "survey", "key": "photograph", "expression": "survey.get('photograph')" }, { "line": 888, "receiver": "photo", "key": "filename", "expression": "photo.get('filename')" }, { "line": 883, "receiver": "photo", "key": "copyright", "expression": "photo.get('copyright', 'NPOR')" }, { "line": 892, "receiver": "photo", "key": "copyright", "expression": "photo.get('copyright', 'NPOR')" } ], "html_queries": [], "dictionary_entries": [ { "line": 894, "key": "images", "value_expression": "images" }, { "line": 883, "key": "copyright", "value_expression": "photo.get('copyright', 'NPOR')" }, { "line": 891, "key": "link", "value_expression": "f'https://npor.org.uk/photographs/{filename}'" }, { "line": 892, "key": "copyright", "value_expression": "photo.get('copyright', 'NPOR')" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._extract_specifications", "path": "aggregator/npor_crawler.py", "line_start": 771, "line_end": 853, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [ { "line": 773, "receiver": "survey", "key": "manuals", "expression": "survey.get('manuals', '0')" }, { "line": 774, "receiver": "survey", "key": "stops", "expression": "survey.get('stops', '0')" }, { "line": 776, "receiver": "survey", "key": "organ", "expression": "survey.get('organ', {})" }, { "line": 781, "receiver": "organ_info", "key": "departments", "expression": "organ_info.get('departments', [])" }, { "line": 823, "receiver": "organ_info", "key": "consoles", "expression": "organ_info.get('consoles', [])" }, { "line": 832, "receiver": "organ_info", "key": "accessories", "expression": "organ_info.get('accessories', [])" }, { "line": 794, "receiver": "dept", "key": "name", "expression": "dept.get('name', 'Unknown')" }, { "line": 799, "receiver": "dept", "key": "key_action", "expression": "dept.get('key_action')" }, { "line": 804, "receiver": "dept", "key": "stops", "expression": "dept.get('stops', [])" }, { "line": 814, "receiver": "dept", "key": "compass_low", "expression": "dept.get('compass_low', '')" }, { "line": 815, "receiver": "dept", "key": "compass_high", "expression": "dept.get('compass_high', '')" }, { "line": 816, "receiver": "dept", "key": "key_action", "expression": "dept.get('key_action', '')" }, { "line": 817, "receiver": "dept", "key": "stop_action", "expression": "dept.get('stop_action', '')" }, { "line": 807, "receiver": "stop", "key": "name", "expression": "stop.get('name', '')" }, { "line": 808, "receiver": "stop", "key": "pitch", "expression": "stop.get('pitch', '')" }, { "line": 809, "receiver": "stop", "key": "remarks", "expression": "stop.get('remarks', '')" }, { "line": 827, "receiver": "console", "key": "couplers", "expression": "console.get('couplers', [])" } ], "html_queries": [], "dictionary_entries": [ { "line": 846, "key": "manual_count", "value_expression": "str(manual_count)" }, { "line": 847, "key": "stop_count", "value_expression": "str(stop_count)" }, { "line": 848, "key": "traction_key", "value_expression": "traction_key" }, { "line": 849, "key": "raw_spec", "value_expression": "' '.join(raw_spec_parts)" }, { "line": 850, "key": "manuals", "value_expression": "manuals_detailed" }, { "line": 851, "key": "couplers", "value_expression": "couplers" }, { "line": 852, "key": "accessories", "value_expression": "accessories" }, { "line": 813, "key": "name", "value_expression": "dept_name" }, { "line": 814, "key": "compass_low", "value_expression": "dept.get('compass_low', '')" }, { "line": 815, "key": "compass_high", "value_expression": "dept.get('compass_high', '')" }, { "line": 816, "key": "key_action", "value_expression": "dept.get('key_action', '')" }, { "line": 817, "key": "stop_action", "value_expression": "dept.get('stop_action', '')" }, { "line": 818, "key": "stops", "value_expression": "stops_data" }, { "line": 807, "key": "name", "value_expression": "stop.get('name', '')" }, { "line": 808, "key": "pitch", "value_expression": "stop.get('pitch', '')" }, { "line": 809, "key": "remarks", "value_expression": "stop.get('remarks', '')" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._fetch_survey", "path": "aggregator/npor_crawler.py", "line_start": 173, "line_end": 185, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_survey_url" ], "literal_get_calls": [ { "line": 182, "receiver": "survey_data", "key": "rec_index", "expression": "survey_data.get('rec_index')" }, { "line": 184, "receiver": "survey_data", "key": "rec_index", "expression": "survey_data.get('rec_index')" } ], "html_queries": [], "dictionary_entries": [ { "line": 176, "key": "Referer", "value_expression": "'https://npor.org.uk/'" }, { "line": 177, "key": "Origin", "value_expression": "'https://npor.org.uk'" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._normalize_survey_ids", "path": "aggregator/npor_crawler.py", "line_start": 111, "line_end": 129, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler._reference_from_survey", "path": "aggregator/npor_crawler.py", "line_start": 187, "line_end": 202, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_extract_main_builder", "_survey_url" ], "literal_get_calls": [ { "line": 188, "receiver": "survey_data", "key": "building", "expression": "survey_data.get('building', {})" }, { "line": 191, "receiver": "survey_data", "key": "rec_index", "expression": "survey_data.get('rec_index')" }, { "line": 194, "receiver": "building", "key": "historic_county", "expression": "building.get('historic_county', '')" }, { "line": 195, "receiver": "building", "key": "town", "expression": "building.get('town', '')" }, { "line": 196, "receiver": "building", "key": "name", "expression": "building.get('name', '')" }, { "line": 198, "receiver": "survey_data", "key": "survey_date", "expression": "survey_data.get('survey_date', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 193, "key": "npor_id", "value_expression": "identifier" }, { "line": 194, "key": "historic_county", "value_expression": "building.get('historic_county', '')" }, { "line": 195, "key": "town", "value_expression": "building.get('town', '')" }, { "line": 196, "key": "building", "value_expression": "building.get('name', '')" }, { "line": 197, "key": "builder", "value_expression": "self._extract_main_builder(survey_data)" }, { "line": 198, "key": "year", "value_expression": "survey_data.get('survey_date', '')" }, { "line": 199, "key": "url", "value_expression": "self._survey_url(identifier)" }, { "line": 200, "key": "status", "value_expression": "'initiated'" }, { "line": 201, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler._reference_location_conflicts", "path": "aggregator/npor_crawler.py", "line_start": 608, "line_end": 622, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 616, "key": "field", "value_expression": "reference_key" }, { "line": 617, "key": "detail_value", "value_expression": "detail_value" }, { "line": 618, "key": "reference_value", "value_expression": "reference_value" }, { "line": 619, "key": "resolution", "value_expression": "'detail retained; search reference preserved for review'" } ], "subscript_assignments": [] }, { "symbol": "NPORCrawler.crawl", "path": "aggregator/npor_crawler.py", "line_start": 87, "line_end": 109, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_crawl_all", "_crawl_single_county", "_crawl_survey_ids", "_update_scrape_status" ], "literal_get_calls": [ { "line": 94, "receiver": "kwargs", "key": "historic_county", "expression": "kwargs.get('historic_county')" }, { "line": 95, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 93, "receiver": "kwargs", "key": "survey_ids", "expression": "kwargs.get('survey_ids')" }, { "line": 93, "receiver": "kwargs", "key": "npor_ids", "expression": "kwargs.get('npor_ids')" }, { "line": 93, "receiver": "kwargs", "key": "ids", "expression": "kwargs.get('ids')" }, { "line": 98, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NPORCrawler.transform", "path": "aggregator/npor_crawler.py", "line_start": 537, "line_end": 605, "inherited": false, "file_sha256": "37fed2b419954637942f223c61fbea73a3259acc101ba69f7293d6c734b2b525", "self_calls": [ "_as_dict_list", "_extract_descriptions", "_extract_events", "_extract_involved", "_extract_literature", "_extract_locations", "_extract_media", "_extract_specifications", "_reference_location_conflicts", "_survey_url" ], "literal_get_calls": [ { "line": 540, "receiver": "survey", "key": "building", "expression": "survey.get('building')" }, { "line": 542, "receiver": "survey", "key": "reference_context", "expression": "survey.get('reference_context', {})" }, { "line": 545, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 540, "receiver": "survey", "key": "building", "expression": "survey.get('building')" }, { "line": 542, "receiver": "survey", "key": "reference_context", "expression": "survey.get('reference_context')" }, { "line": 559, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 560, "receiver": "survey", "key": "survey_date", "expression": "survey.get('survey_date', '')" }, { "line": 561, "receiver": "survey", "key": "record_type", "expression": "survey.get('record_type', '')" }, { "line": 595, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 554, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 555, "receiver": "building_meta", "key": "building_index", "expression": "building_meta.get('building_index', '')" }, { "line": 556, "receiver": "building_meta", "key": "place_index", "expression": "building_meta.get('place_index', '')" }, { "line": 562, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 568, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 569, "receiver": "survey", "key": "survey_date", "expression": "survey.get('survey_date', '')" }, { "line": 571, "receiver": "survey", "key": "record_type", "expression": "survey.get('record_type', '')" }, { "line": 572, "receiver": "building_meta", "key": "building_index", "expression": "building_meta.get('building_index', '')" }, { "line": 573, "receiver": "building_meta", "key": "place_index", "expression": "building_meta.get('place_index', '')" }, { "line": 596, "receiver": "survey", "key": "url", "expression": "survey.get('url')" }, { "line": 570, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 576, "receiver": "survey", "key": "history", "expression": "survey.get('history')" }, { "line": 577, "receiver": "survey", "key": "updates", "expression": "survey.get('updates')" }, { "line": 596, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 562, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 576, "receiver": "survey", "key": "history", "expression": "survey.get('history')" }, { "line": 596, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" }, { "line": 570, "receiver": "survey", "key": "rec_index", "expression": "survey.get('rec_index')" } ], "html_queries": [], "dictionary_entries": [ { "line": 545, "key": "npor_id", "value_expression": "survey.get('rec_index')" }, { "line": 546, "key": "locations", "value_expression": "self._extract_locations(survey)" }, { "line": 547, "key": "events", "value_expression": "self._extract_events(survey)" }, { "line": 548, "key": "involved", "value_expression": "self._extract_involved(survey)" }, { "line": 549, "key": "specifications", "value_expression": "self._extract_specifications(survey)" }, { "line": 550, "key": "media", "value_expression": "self._extract_media(survey)" }, { "line": 551, "key": "literature", "value_expression": "self._extract_literature(survey)" }, { "line": 552, "key": "descriptions", "value_expression": "self._extract_descriptions(survey)" }, { "line": 553, "key": "other_identifiers", "value_expression": "[{'type': 'npor_survey_id', 'value': survey.get('rec_index')}, {'type': 'npor_building_index', 'value': building_meta.get('building_index', '')}, {'type': 'npor_place_index', 'value': building_meta.get('place_index', '')}]" }, { "line": 558, "key": "source_listing", "value_expression": "{'survey_id': survey.get('rec_index'), 'survey_date': survey.get('survey_date', ''), 'record_type': survey.get('record_type', ''), 'api_url': f\"{self.API_BASE_SURVEY}{survey.get('rec_index')}\" if survey.get('rec_index') else ''}" }, { "line": 559, "key": "survey_id", "value_expression": "survey.get('rec_index')" }, { "line": 560, "key": "survey_date", "value_expression": "survey.get('survey_date', '')" }, { "line": 561, "key": "record_type", "value_expression": "survey.get('record_type', '')" }, { "line": 562, "key": "api_url", "value_expression": "f\"{self.API_BASE_SURVEY}{survey.get('rec_index')}\" if survey.get('rec_index') else ''" }, { "line": 588, "key": "warnings", "value_expression": "['detail_location_conflicts_with_search_reference']" }, { "line": 589, "key": "conflicts", "value_expression": "conflicts" }, { "line": 554, "key": "type", "value_expression": "'npor_survey_id'" }, { "line": 554, "key": "value", "value_expression": "survey.get('rec_index')" }, { "line": 555, "key": "type", "value_expression": "'npor_building_index'" }, { "line": 555, "key": "value", "value_expression": "building_meta.get('building_index', '')" }, { "line": 556, "key": "type", "value_expression": "'npor_place_index'" }, { "line": 556, "key": "value", "value_expression": "building_meta.get('place_index', '')" }, { "line": 566, "key": "id_system", "value_expression": "'survey IDs are a single uppercase letter plus five digits, e.g. D00002'" }, { "line": 567, "key": "survey_record_is_source_event", "value_expression": "True" }, { "line": 568, "key": "rec_index", "value_expression": "survey.get('rec_index')" }, { "line": 569, "key": "survey_date", "value_expression": "survey.get('survey_date', '')" }, { "line": 570, "key": "api_url", "value_expression": "f\"{self.API_BASE_SURVEY}{survey.get('rec_index')}\" if survey.get('rec_index') else ''" }, { "line": 571, "key": "record_type", "value_expression": "survey.get('record_type', '')" }, { "line": 572, "key": "building_index", "value_expression": "building_meta.get('building_index', '')" }, { "line": 573, "key": "place_index", "value_expression": "building_meta.get('place_index', '')" }, { "line": 574, "key": "reference_context", "value_expression": "reference_context" }, { "line": 575, "key": "reference_conflicts", "value_expression": "conflicts" }, { "line": 576, "key": "history", "value_expression": "survey.get('history') if isinstance(survey.get('history'), dict) else {}" }, { "line": 577, "key": "updates", "value_expression": "self._as_dict_list(survey.get('updates'))" }, { "line": 578, "key": "transform_policies", "value_expression": "{'events': 'npor-builder-assertion-events-v1', 'literature': 'npor-archive-sources-v1', 'descriptions': 'npor-source-context-v1'}" }, { "line": 579, "key": "events", "value_expression": "'npor-builder-assertion-events-v1'" }, { "line": 580, "key": "literature", "value_expression": "'npor-archive-sources-v1'" }, { "line": 581, "key": "descriptions", "value_expression": "'npor-source-context-v1'" } ], "subscript_assignments": [ { "line": 604, "value_line": 604, "key": "normalized_at", "target_expression": "record['provenance']['normalized_at']", "value_expression": "self.RELEASE_03_NORMALIZED_AT" }, { "line": 587, "value_line": 587, "key": "quality", "target_expression": "transformed['quality']", "value_expression": "{'warnings': ['detail_location_conflicts_with_search_reference'], 'conflicts': conflicts}" } ] } ], "selected_rule_rows": [ { "symbol": "NPORCrawler._extract_specifications", "path": "aggregator/npor_crawler.py", "line": 847, "key": "stop_count", "value_expression": "str(stop_count)" }, { "symbol": "NPORCrawler._extract_specifications", "path": "aggregator/npor_crawler.py", "line": 846, "key": "manual_count", "value_expression": "str(manual_count)" }, { "symbol": "NPORCrawler._extract_events", "path": "aggregator/npor_crawler.py", "line": 717, "key": "dates", "value_expression": "[date] if date else []" }, { "symbol": "NPORCrawler.transform", "path": "aggregator/npor_crawler.py", "line": 561, "key": "record_type", "value_expression": "survey.get('record_type', '')" }, { "symbol": "NPORCrawler._extract_locations", "path": "aggregator/npor_crawler.py", "line": 637, "key": "country", "value_expression": "'United Kingdom'" }, { "symbol": "NPORCrawler._extract_literature", "path": "aggregator/npor_crawler.py", "line": 920, "key": "raw_str", "value_expression": "'; '.join(citation_parts)" } ], "bounded_code_passages": [ { "path": "aggregator/npor_crawler.py", "symbol": "NPORCrawler._extract_specifications", "line_start": 844, "line_end": 852, "text": " \n return {\n \"manual_count\": str(manual_count),\n \"stop_count\": str(stop_count),\n \"traction_key\": traction_key,\n \"raw_spec\": \" \".join(raw_spec_parts),\n \"manuals\": manuals_detailed,\n \"couplers\": couplers,\n \"accessories\": accessories" }, { "path": "aggregator/npor_crawler.py", "symbol": "NPORCrawler._extract_specifications", "line_start": 843, "line_end": 851, "text": " # For 'raw_spec' column, simpler is usually better, but for JSON we have 'manuals_detailed'.\n \n return {\n \"manual_count\": str(manual_count),\n \"stop_count\": str(stop_count),\n \"traction_key\": traction_key,\n \"raw_spec\": \" \".join(raw_spec_parts),\n \"manuals\": manuals_detailed,\n \"couplers\": couplers," }, { "path": "aggregator/npor_crawler.py", "symbol": "NPORCrawler._extract_events", "line_start": 714, "line_end": 722, "text": " event = {\n \"id\": f\"builder-assertion-{index + 1}\",\n \"type\": event_type,\n \"dates\": [date] if date else [],\n \"activities\": activities,\n \"involved\": [{\"name\": name, \"activities\": activities}] if name else [],\n \"description\": work or None,\n \"source_native\": {\n \"section\": \"builders\"," }, { "path": "aggregator/npor_crawler.py", "symbol": "NPORCrawler.transform", "line_start": 558, "line_end": 566, "text": " \"source_listing\": {\n \"survey_id\": survey.get(\"rec_index\"),\n \"survey_date\": survey.get(\"survey_date\", \"\"),\n \"record_type\": survey.get(\"record_type\", \"\"),\n \"api_url\": f\"{self.API_BASE_SURVEY}{survey.get('rec_index')}\" if survey.get(\"rec_index\") else \"\",\n },\n \"extensions\": {\n \"npor\": {\n \"id_system\": \"survey IDs are a single uppercase letter plus five digits, e.g. D00002\"," }, { "path": "aggregator/npor_crawler.py", "symbol": "NPORCrawler._extract_locations", "line_start": 634, "line_end": 639, "text": " \"building\": building_info.get(\"name\", \"\"),\n \"address\": building_info.get(\"address\", \"\"),\n \"postcode\": building_info.get(\"postcode\", \"\"),\n \"country\": \"United Kingdom\",\n \"region\": \"\" # Region is not explicitly in NPOR data usually\n }]" }, { "path": "aggregator/npor_crawler.py", "symbol": "NPORCrawler._extract_literature", "line_start": 917, "line_end": 925, "text": " literature.append(\n {\n \"type\": f\"npor_archive_{source_type}\",\n \"raw_str\": \"; \".join(citation_parts),\n \"author\": native.get(\"author\", \"\"),\n \"title\": native.get(\"title\") or native.get(\"document\", \"\"),\n \"date\": native.get(\"date\", \"\"),\n \"pages\": native.get(\"page\", \"\"),\n \"source_native\": native," } ] } }, { "catalogue_id": "S025", "source_key": "nycago", "name": "NYC AGO", "selection_role": "registered_source", "parent_source_key": null, "attribution": "New York City Chapter, American Guild of Organists", "scope_description": "Present and former organs in all five New York City boroughs", "documentary_unit": "Building or venue page containing organ histories", "source_description": "The project's official index describes an ongoing record of organs installed in the five boroughs. It distinguishes non-extant entries and indicates whether a specification is available. Direct requests were refused, so the content check used an indexed official page.", "source_description_references": [ "indexed_nycago", "nycago_catalogue", "record_nycago" ], "interpretation": "One building page may describe successive instruments. Borough, venue and installation history should therefore be retained when reconciling identifiers. The indexed evidence is older than this package's retrieval date.", "rights_observation": "No collection-wide reuse permission was established from the consulted evidence.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "indexed_nycago", "nycago_catalogue", "record_nycago" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_nycago", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "access_finding": "The selected detail URL returned HTTP 403. Its native content was not examined." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler discovers entries from the NYC AGO directory and retains listing title, category and borough with the detail HTML.", "transformation": "The inspected transform obtains the venue from a marked span or bold element, scans marked builder spans and the following text for a four-digit year, and collects image references. It creates building/location, builder-history and source-credit structures. Its specifications object explicitly states that further parsing logic is required; it does not implement a detailed stop-table projection in this entry point.", "limitations": "The archive can contain detailed stoplists even when this transform does not structure them. Years near builder names and venue-level pages do not settle the identity of successive organs. Image collection at this level is broad and does not by itself distinguish instrument photographs from every other image." }, "implementation": { "class": "NycagoCrawler", "class_line": 19, "path": "aggregator/nycago_crawler.py", "inheritance": [ "NycagoCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "NycagoCrawler._category_for_index_cell", "path": "aggregator/nycago_crawler.py", "line_start": 212, "line_end": 218, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NycagoCrawler._fetch_organ_details", "path": "aggregator/nycago_crawler.py", "line_start": 166, "line_end": 174, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NycagoCrawler._iter_index_entries", "path": "aggregator/nycago_crawler.py", "line_start": 220, "line_end": 256, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [ "_category_for_index_cell", "_is_organ_header", "_nycago_id_from_detail_link" ], "literal_get_calls": [ { "line": 233, "receiver": "link_tag", "key": "href", "expression": "link_tag.get('href')" } ], "html_queries": [ { "line": 228, "expression": "soup.find_all('td')" }, { "line": 232, "expression": "td.find('a')" } ], "dictionary_entries": [ { "line": 247, "key": "id", "value_expression": "self._nycago_id_from_detail_link(detail_link, borough_code)" }, { "line": 248, "key": "title", "value_expression": "title" }, { "line": 249, "key": "category", "value_expression": "category" }, { "line": 250, "key": "link", "value_expression": "detail_link" }, { "line": 251, "key": "borough", "value_expression": "borough_label" }, { "line": 252, "key": "borough_code", "value_expression": "borough_code" } ], "subscript_assignments": [] }, { "symbol": "NycagoCrawler._normalize_borough_key", "path": "aggregator/nycago_crawler.py", "line_start": 176, "line_end": 177, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NycagoCrawler._nycago_id_from_detail_link", "path": "aggregator/nycago_crawler.py", "line_start": 206, "line_end": 210, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "NycagoCrawler.crawl", "path": "aggregator/nycago_crawler.py", "line_start": 258, "line_end": 293, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [ "_fetch_organ_details", "_iter_index_entries", "_resolve_boroughs", "_save_cache_reference" ], "literal_get_calls": [ { "line": 264, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 267, "receiver": "kwargs", "key": "borough", "expression": "kwargs.get('borough')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 284, "value_line": 284, "key": "html", "target_expression": "entry['html']", "value_expression": "detail_html" } ] }, { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "line_start": 334, "line_end": 399, "inherited": false, "file_sha256": "01c3d3ab2673ddff38c2c0f6b52ea347cca0f53dedca1183bf6cd510e1a9e16d", "self_calls": [], "literal_get_calls": [ { "line": 359, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 378, "receiver": "raw_data", "key": "borough", "expression": "raw_data.get('borough')" }, { "line": 386, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 387, "receiver": "raw_data", "key": "category", "expression": "raw_data.get('category', '')" }, { "line": 388, "receiver": "raw_data", "key": "borough", "expression": "raw_data.get('borough', '')" }, { "line": 396, "receiver": "raw_data", "key": "link", "expression": "raw_data.get('link')" }, { "line": 397, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html', '')" }, { "line": 395, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 395, "receiver": "raw_data", "key": "nycago_id", "expression": "raw_data.get('nycago_id')" }, { "line": 363, "receiver": "img", "key": "alt", "expression": "img.get('alt', '')" } ], "html_queries": [ { "line": 344, "expression": "soup.find_all('span', class_='builder')" }, { "line": 358, "expression": "soup.find_all('img')" }, { "line": 340, "expression": "soup.find('span', class_='venue')" }, { "line": 340, "expression": "soup.find('b', class_='venue')" } ], "dictionary_entries": [ { "line": 356, "key": "description", "value_expression": "f'Data for {building}. Full parsing requires additional parser logic.'" }, { "line": 357, "key": "images", "value_expression": "[]" }, { "line": 357, "key": "audio", "value_expression": "[]" }, { "line": 357, "key": "video", "value_expression": "[]" }, { "line": 369, "key": "source_info", "value_expression": "{'name': self.source_name, 'url': raw_data['link'], 'retrieval_date': datetime.utcnow().isoformat() + 'Z'}" }, { "line": 374, "key": "organ_identifiers", "value_expression": "[{'name': 'nycago_id', 'value': raw_data['id']}]" }, { "line": 375, "key": "location", "value_expression": "{'building': building, 'city': 'New York', 'region': raw_data.get('borough'), 'country': 'United States'}" }, { "line": 381, "key": "builder_history", "value_expression": "builder_history" }, { "line": 382, "key": "specifications", "value_expression": "specifications" }, { "line": 383, "key": "media", "value_expression": "media" }, { "line": 384, "key": "sources", "value_expression": "[{'citation': 'The Organ Project, New York City Chapter of the American Guild of Organists.'}]" }, { "line": 385, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', ''), 'category': raw_data.get('category', ''), 'borough': raw_data.get('borough', '')}" }, { "line": 370, "key": "name", "value_expression": "self.source_name" }, { "line": 371, "key": "url", "value_expression": "raw_data['link']" }, { "line": 372, "key": "retrieval_date", "value_expression": "datetime.utcnow().isoformat() + 'Z'" }, { "line": 376, "key": "building", "value_expression": "building" }, { "line": 377, "key": "city", "value_expression": "'New York'" }, { "line": 378, "key": "region", "value_expression": "raw_data.get('borough')" }, { "line": 379, "key": "country", "value_expression": "'United States'" }, { "line": 386, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 387, "key": "category", "value_expression": "raw_data.get('category', '')" }, { "line": 388, "key": "borough", "value_expression": "raw_data.get('borough', '')" }, { "line": 354, "key": "builder_name", "value_expression": "builder_name" }, { "line": 354, "key": "year", "value_expression": "year" }, { "line": 354, "key": "activity", "value_expression": "'Information'" }, { "line": 374, "key": "name", "value_expression": "'nycago_id'" }, { "line": 374, "key": "value", "value_expression": "raw_data['id']" }, { "line": 384, "key": "citation", "value_expression": "'The Organ Project, New York City Chapter of the American Guild of Organists.'" }, { "line": 362, "key": "url", "value_expression": "urljoin(self.base_url, img['src'])" }, { "line": 363, "key": "caption", "value_expression": "img.get('alt', '')" }, { "line": 364, "key": "source", "value_expression": "self.source_name" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "line": 379, "key": "country", "value_expression": "'United States'" }, { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "key": "src", "value_expression": "img.get('src')", "line": 359, "kind": "input read" }, { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "key": "borough", "value_expression": "raw_data.get('borough')", "line": 378, "kind": "input read" }, { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "key": "title", "value_expression": "raw_data.get('title', '')", "line": 386, "kind": "input read" }, { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "key": "category", "value_expression": "raw_data.get('category', '')", "line": 387, "kind": "input read" }, { "symbol": "NycagoCrawler.transform", "path": "aggregator/nycago_crawler.py", "key": "link", "value_expression": "raw_data.get('link')", "line": 396, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/nycago_crawler.py", "symbol": "NycagoCrawler.transform", "line_start": 376, "line_end": 384, "text": " \"building\": building,\n \"city\": \"New York\",\n \"region\": raw_data.get(\"borough\"),\n \"country\": \"United States\",\n },\n \"builder_history\": builder_history,\n \"specifications\": specifications,\n \"media\": media,\n \"sources\": [{\"citation\": \"The Organ Project, New York City Chapter of the American Guild of Organists.\"}]," }, { "path": "aggregator/nycago_crawler.py", "symbol": "NycagoCrawler.transform", "line_start": 356, "line_end": 364, "text": " specifications = {\"description\": f\"Data for {building}. Full parsing requires additional parser logic.\"}\n media = {\"images\": [], \"audio\": [], \"video\": []}\n for img in soup.find_all(\"img\"):\n if img.get(\"src\"):\n media[\"images\"].append(\n {\n \"url\": urljoin(self.base_url, img[\"src\"]),\n \"caption\": img.get(\"alt\", \"\"),\n \"source\": self.source_name," }, { "path": "aggregator/nycago_crawler.py", "symbol": "NycagoCrawler.transform", "line_start": 375, "line_end": 383, "text": " \"location\": {\n \"building\": building,\n \"city\": \"New York\",\n \"region\": raw_data.get(\"borough\"),\n \"country\": \"United States\",\n },\n \"builder_history\": builder_history,\n \"specifications\": specifications,\n \"media\": media," }, { "path": "aggregator/nycago_crawler.py", "symbol": "NycagoCrawler.transform", "line_start": 383, "line_end": 391, "text": " \"media\": media,\n \"sources\": [{\"citation\": \"The Organ Project, New York City Chapter of the American Guild of Organists.\"}],\n \"source_listing\": {\n \"title\": raw_data.get(\"title\", \"\"),\n \"category\": raw_data.get(\"category\", \"\"),\n \"borough\": raw_data.get(\"borough\", \"\"),\n },\n }\n" }, { "path": "aggregator/nycago_crawler.py", "symbol": "NycagoCrawler.transform", "line_start": 384, "line_end": 392, "text": " \"sources\": [{\"citation\": \"The Organ Project, New York City Chapter of the American Guild of Organists.\"}],\n \"source_listing\": {\n \"title\": raw_data.get(\"title\", \"\"),\n \"category\": raw_data.get(\"category\", \"\"),\n \"borough\": raw_data.get(\"borough\", \"\"),\n },\n }\n\n return normalize_transfer_record(" }, { "path": "aggregator/nycago_crawler.py", "symbol": "NycagoCrawler.transform", "line_start": 393, "line_end": 399, "text": " structured_entry,\n source_key=\"nycago\",\n primary_id=raw_data.get(\"id\") or raw_data.get(\"nycago_id\"),\n source_url=raw_data.get(\"link\"),\n raw_content=raw_data.get(\"html\", \"\"),\n content_type=\"text/html\",\n )" } ] } }, { "catalogue_id": "S026", "source_key": "ohs", "name": "OHS Pipe Organ Database", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Organ Historical Society", "scope_description": "Pipe Organ Database", "documentary_unit": "Instrument record with related builders, places and documentary items", "source_description": "The database exposes instrument, builder and location searching alongside stoplists, images and documents. Its home page separately reports totals for these entities and identifies the project as belonging to the Organ Historical Society.", "source_description_references": [ "web_ohs", "ohs", "rendered_record_ohs" ], "interpretation": "The separate counters should remain separate populations. A stoplist may describe a particular date or configuration, and a builder attribution may concern an original instrument or later work. Neither should be collapsed into an undated present-day description.", "rights_observation": "The home page states that its information, images and documents may be used for educational or other purposes. Individual credits and rights still need to accompany reused items.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "web_ohs", "ohs", "rendered_record_ohs" ], "observation_date": "2026-09-10", "selected_document": { "reference": "rendered_record_ohs", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Relational instrument page with separately identified location, builder, stoplist and console records", "repeated_information": "Selectable stoplists and consoles; divisions contain stop rows. Documents, audio and notes are separate collections.", "date_and_state_distinctions": "Instrument year, stoplist publication date, submission date and dated editorial notes are separate.", "interpretive_limit": "A stoplist or console is a child record with its own evidence. Zero audio items in this example is not a site-wide absence.", "observations": [ "Building, room, street address, city and location-record link.", "A map locates the installation.", "Named manufacturer linked to a builder record, with opus and year.", "Dated notes record later alterations and information updates.", "Selectable stoplist with named divisions, pitches and pipe quantities.", "Console record gives manual and pedal note counts.", "Key and stop actions are separate console attributes.", "Separate console record describes position, design and controls.", "Extant and playable are explicit instrument states.", "Three documents are counted, and the stoplist has an attributed documentary source.", "Location, builder and stoplist identifiers have separate routes.", "Original stoplist publication, receipt and later editorial dates are distinguished." ], "uninspected_or_unfilled_material": [ "Image section and category filters are exposed; image payloads were not inspected.", "The selected instrument explicitly reports zero audio items.", "The selected instrument explicitly reports zero blower records." ], "narrative": "Instrument 1 links to separately identified location, builder, stoplist and console records. The selected stoplist has attribution and documentary dates; the console supplies its own technical information. The interface explicitly counted no audio items and no blower records for this example. Those empty collections do not establish that the database lacks those record types or that the instrument has no wind supply." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "An instrument request is supplemented by related endpoints and separately fetched stoplist resources. The crawler has a dedicated branch for uploaded text stoplists and a different branch for structured stoplist responses. Resource URLs and hashes accompany text parsing.", "transformation": "Instrument, location, builder, console, stoplist and media material enter different routines. The stoplist parser retains raw text, stops, couplers, accessories and unresolved lines. Console or division compass context may be attached to stop entries with a named provenance field. Current and original builder participation is delegated to a source-specific helper. The resulting instrument transfer is an assembly of several documentary objects, not a verbatim copy of one page.", "limitations": "Failure of a related endpoint can be represented by an empty array; failure to fetch a stoplist is logged and the loop continues. Absence in the assembled transfer therefore does not prove absence upstream. A compass-based note count is contextual assignment, not a measured pipe count. Stoplist-resource dates and instrument chronology need to remain separately attributable." }, "implementation": { "class": "OhsCrawler", "class_line": 17, "path": "aggregator/ohs_crawler.py", "inheritance": [ "OhsCrawler", "BaseCrawler" ], "configuration": { "RELATED_ENDPOINTS": { "value": { "images": "images", "webpages": "webpages", "references": "references", "consoles": "organConsoles", "documents": "documents", "audio": "audio", "notes": "notes" }, "path": "aggregator/ohs_crawler.py", "line": 23, "class": "OhsCrawler" } }, "methods": [ { "symbol": "OhsCrawler._build_audio", "path": "aggregator/ohs_crawler.py", "line_start": 753, "line_end": 759, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 756, "receiver": "audio", "key": "filename", "expression": "audio.get('filename')" }, { "line": 758, "receiver": "audio", "key": "name", "expression": "audio.get('name')" } ], "html_queries": [], "dictionary_entries": [ { "line": 758, "key": "url", "value_expression": "f'{self.API_BASE}/uploads/audio/{filename}'" }, { "line": 758, "key": "label", "value_expression": "audio.get('name') or ''" }, { "line": 758, "key": "raw", "value_expression": "audio" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_descriptions", "path": "aggregator/ohs_crawler.py", "line_start": 790, "line_end": 797, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_strip_html" ], "literal_get_calls": [ { "line": 792, "receiver": "item", "key": "approvedNote", "expression": "item.get('approvedNote')" }, { "line": 795, "receiver": "note", "key": "content", "expression": "note.get('content')" } ], "html_queries": [], "dictionary_entries": [ { "line": 793, "key": "type", "value_expression": "'approval_note'" }, { "line": 793, "key": "text", "value_expression": "self._strip_html(item['approvedNote'])" }, { "line": 796, "key": "type", "value_expression": "'note'" }, { "line": 796, "key": "text", "value_expression": "self._strip_html(note['content'])" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_documents", "path": "aggregator/ohs_crawler.py", "line_start": 761, "line_end": 767, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 764, "receiver": "document", "key": "filename", "expression": "document.get('filename')" }, { "line": 766, "receiver": "document", "key": "name", "expression": "document.get('name')" } ], "html_queries": [], "dictionary_entries": [ { "line": 766, "key": "url", "value_expression": "f'{self.API_BASE}/uploads/documents/{filename}'" }, { "line": 766, "key": "label", "value_expression": "document.get('name') or ''" }, { "line": 766, "key": "raw", "value_expression": "document" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_events", "path": "aggregator/ohs_crawler.py", "line_start": 528, "line_end": 546, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_format_original_opus", "_format_original_year", "_format_year", "_title_for_item" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 533, "key": "type", "value_expression": "'construction_or_rebuild'" }, { "line": 533, "key": "date", "value_expression": "self._format_year(item)" }, { "line": 533, "key": "description", "value_expression": "self._title_for_item(item)" }, { "line": 537, "key": "type", "value_expression": "'original_construction'" }, { "line": 538, "key": "date", "value_expression": "self._format_original_year(item)" }, { "line": 539, "key": "description", "value_expression": "self._format_original_opus(item)" } ], "subscript_assignments": [ { "line": 545, "value_line": 545, "key": "involved", "target_expression": "event['involved']", "value_expression": "[participant]" } ] }, { "symbol": "OhsCrawler._build_images", "path": "aggregator/ohs_crawler.py", "line_start": 735, "line_end": 751, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 738, "receiver": "image", "key": "filename", "expression": "image.get('filename')" }, { "line": 738, "receiver": "image", "key": "fullThumbFilename", "expression": "image.get('fullThumbFilename')" }, { "line": 738, "receiver": "image", "key": "thumbFilename", "expression": "image.get('thumbFilename')" }, { "line": 748, "receiver": "image", "key": "id", "expression": "image.get('id')" }, { "line": 744, "receiver": "image", "key": "thumbFilename", "expression": "image.get('thumbFilename')" }, { "line": 745, "receiver": "image", "key": "name", "expression": "image.get('name')" }, { "line": 745, "receiver": "image", "key": "source", "expression": "image.get('source')" }, { "line": 746, "receiver": "image", "key": "source", "expression": "image.get('source')" }, { "line": 747, "receiver": "image", "key": "dateTaken", "expression": "image.get('dateTaken')" }, { "line": 747, "receiver": "image", "key": "dateTakenApprox", "expression": "image.get('dateTakenApprox')" }, { "line": 744, "receiver": "image", "key": "thumbFilename", "expression": "image.get('thumbFilename')" } ], "html_queries": [], "dictionary_entries": [ { "line": 743, "key": "link", "value_expression": "f'{self.API_BASE}/uploads/images/{filename}'" }, { "line": 744, "key": "thumbnail", "value_expression": "f\"{self.API_BASE}/uploads/images/{image.get('thumbFilename')}\" if image.get('thumbFilename') else ''" }, { "line": 745, "key": "caption", "value_expression": "image.get('name') or image.get('source') or ''" }, { "line": 746, "key": "source", "value_expression": "image.get('source') or self.source_name" }, { "line": 747, "key": "date", "value_expression": "image.get('dateTaken') or image.get('dateTakenApprox') or ''" }, { "line": 748, "key": "source_id", "value_expression": "image.get('id')" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_involved", "path": "aggregator/ohs_crawler.py", "line_start": 502, "line_end": 526, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_format_original_year", "_format_year" ], "literal_get_calls": [ { "line": 504, "receiver": "item", "key": "builder", "expression": "item.get('builder')" }, { "line": 505, "receiver": "builder", "key": "name", "expression": "builder.get('name')" }, { "line": 505, "receiver": "item", "key": "submissionBuilder", "expression": "item.get('submissionBuilder')" }, { "line": 515, "receiver": "item", "key": "originalBuilder", "expression": "item.get('originalBuilder')" }, { "line": 516, "receiver": "original_builder", "key": "name", "expression": "original_builder.get('name')" }, { "line": 516, "receiver": "item", "key": "submissionOriginalBuilder", "expression": "item.get('submissionOriginalBuilder')" }, { "line": 512, "receiver": "builder", "key": "id", "expression": "builder.get('id')" }, { "line": 523, "receiver": "original_builder", "key": "id", "expression": "original_builder.get('id')" }, { "line": 508, "receiver": "builder", "key": "name", "expression": "builder.get('name')" }, { "line": 508, "receiver": "item", "key": "submissionBuilder", "expression": "item.get('submissionBuilder')" }, { "line": 511, "receiver": "builder", "key": "name", "expression": "builder.get('name')" }, { "line": 511, "receiver": "item", "key": "submissionBuilder", "expression": "item.get('submissionBuilder')" }, { "line": 519, "receiver": "original_builder", "key": "name", "expression": "original_builder.get('name')" }, { "line": 519, "receiver": "item", "key": "submissionOriginalBuilder", "expression": "item.get('submissionOriginalBuilder')" }, { "line": 522, "receiver": "original_builder", "key": "name", "expression": "original_builder.get('name')" }, { "line": 522, "receiver": "item", "key": "submissionOriginalBuilder", "expression": "item.get('submissionOriginalBuilder')" } ], "html_queries": [], "dictionary_entries": [ { "line": 508, "key": "name", "value_expression": "builder.get('name') or item.get('submissionBuilder')" }, { "line": 509, "key": "activities", "value_expression": "['builder']" }, { "line": 510, "key": "dates", "value_expression": "[self._format_year(item)] if self._format_year(item) else []" }, { "line": 511, "key": "raw_str", "value_expression": "builder.get('name') or item.get('submissionBuilder')" }, { "line": 512, "key": "source_id", "value_expression": "builder.get('id')" }, { "line": 519, "key": "name", "value_expression": "original_builder.get('name') or item.get('submissionOriginalBuilder')" }, { "line": 520, "key": "activities", "value_expression": "['original_builder']" }, { "line": 521, "key": "dates", "value_expression": "[self._format_original_year(item)] if self._format_original_year(item) else []" }, { "line": 522, "key": "raw_str", "value_expression": "original_builder.get('name') or item.get('submissionOriginalBuilder')" }, { "line": 523, "key": "source_id", "value_expression": "original_builder.get('id')" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_links", "path": "aggregator/ohs_crawler.py", "line_start": 770, "line_end": 778, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 777, "receiver": "webpage", "key": "url", "expression": "webpage.get('url')" }, { "line": 773, "receiver": "webpage", "key": "url", "expression": "webpage.get('url')" }, { "line": 774, "receiver": "webpage", "key": "name", "expression": "webpage.get('name')" }, { "line": 774, "receiver": "webpage", "key": "description", "expression": "webpage.get('description')" }, { "line": 774, "receiver": "webpage", "key": "url", "expression": "webpage.get('url')" } ], "html_queries": [], "dictionary_entries": [ { "line": 773, "key": "url", "value_expression": "webpage.get('url') or ''" }, { "line": 774, "key": "label", "value_expression": "webpage.get('name') or webpage.get('description') or webpage.get('url') or ''" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_literature", "path": "aggregator/ohs_crawler.py", "line_start": 781, "line_end": 788, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 784, "receiver": "ref", "key": "name", "expression": "ref.get('name')" }, { "line": 784, "receiver": "ref", "key": "description", "expression": "ref.get('description')" }, { "line": 784, "receiver": "ref", "key": "source", "expression": "ref.get('source')" } ], "html_queries": [], "dictionary_entries": [ { "line": 787, "key": "raw_str", "value_expression": "raw" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_location", "path": "aggregator/ohs_crawler.py", "line_start": 477, "line_end": 500, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 478, "receiver": "item", "key": "location", "expression": "item.get('location')" }, { "line": 497, "receiver": "location", "key": "latitude", "expression": "location.get('latitude')" }, { "line": 498, "receiver": "location", "key": "longitude", "expression": "location.get('longitude')" }, { "line": 480, "receiver": "location", "key": "name", "expression": "location.get('name')" }, { "line": 480, "receiver": "item", "key": "submissionLocation", "expression": "item.get('submissionLocation')" }, { "line": 481, "receiver": "item", "key": "room", "expression": "item.get('room')" }, { "line": 482, "receiver": "location", "key": "address", "expression": "location.get('address')" }, { "line": 483, "receiver": "location", "key": "city", "expression": "location.get('city')" }, { "line": 484, "receiver": "location", "key": "state", "expression": "location.get('state')" }, { "line": 485, "receiver": "location", "key": "country", "expression": "location.get('country')" }, { "line": 488, "receiver": "item", "key": "extant", "expression": "item.get('extant')" }, { "line": 489, "receiver": "location", "key": "country", "expression": "location.get('country')" }, { "line": 490, "receiver": "location", "key": "state", "expression": "location.get('state')" }, { "line": 491, "receiver": "location", "key": "state", "expression": "location.get('state')" }, { "line": 492, "receiver": "location", "key": "city", "expression": "location.get('city')" }, { "line": 493, "receiver": "location", "key": "city", "expression": "location.get('city')" }, { "line": 494, "receiver": "location", "key": "name", "expression": "location.get('name')" }, { "line": 494, "receiver": "item", "key": "submissionLocation", "expression": "item.get('submissionLocation')" }, { "line": 495, "receiver": "item", "key": "room", "expression": "item.get('room')" }, { "line": 496, "receiver": "location", "key": "address", "expression": "location.get('address')" } ], "html_queries": [], "dictionary_entries": [ { "line": 488, "key": "status", "value_expression": "'current_location' if item.get('extant') else 'former_or_planned_location'" }, { "line": 489, "key": "country", "value_expression": "location.get('country') or ''" }, { "line": 490, "key": "region", "value_expression": "location.get('state') or ''" }, { "line": 491, "key": "state", "value_expression": "location.get('state') or ''" }, { "line": 492, "key": "town", "value_expression": "location.get('city') or ''" }, { "line": 493, "key": "city", "value_expression": "location.get('city') or ''" }, { "line": 494, "key": "building", "value_expression": "location.get('name') or item.get('submissionLocation') or ''" }, { "line": 495, "key": "room", "value_expression": "item.get('room') or ''" }, { "line": 496, "key": "address", "value_expression": "location.get('address') or ''" }, { "line": 497, "key": "latitude", "value_expression": "location.get('latitude')" }, { "line": 498, "key": "longitude", "value_expression": "location.get('longitude')" }, { "line": 499, "key": "raw_str", "value_expression": "', '.join([str(part) for part in parts if part not in (None, '')])" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_other_identifiers", "path": "aggregator/ohs_crawler.py", "line_start": 799, "line_end": 815, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 805, "receiver": "item", "key": "originalId", "expression": "item.get('originalId')" }, { "line": 807, "receiver": "item", "key": "originalInstrumentId", "expression": "item.get('originalInstrumentId')" }, { "line": 809, "receiver": "item", "key": "futureInstrumentId", "expression": "item.get('futureInstrumentId')" }, { "line": 801, "receiver": "item", "key": "id", "expression": "item.get('id')" }, { "line": 802, "receiver": "item", "key": "locationId", "expression": "item.get('locationId')" }, { "line": 803, "receiver": "item", "key": "builderId", "expression": "item.get('builderId')" }, { "line": 806, "receiver": "item", "key": "originalId", "expression": "item.get('originalId')" }, { "line": 808, "receiver": "item", "key": "originalInstrumentId", "expression": "item.get('originalInstrumentId')" }, { "line": 810, "receiver": "item", "key": "futureInstrumentId", "expression": "item.get('futureInstrumentId')" }, { "line": 814, "receiver": "entry", "key": "value", "expression": "entry.get('value')" } ], "html_queries": [], "dictionary_entries": [ { "line": 801, "key": "type", "value_expression": "'ohs_current_id'" }, { "line": 801, "key": "value", "value_expression": "item.get('id')" }, { "line": 802, "key": "type", "value_expression": "'ohs_location_id'" }, { "line": 802, "key": "value", "value_expression": "item.get('locationId')" }, { "line": 803, "key": "type", "value_expression": "'ohs_builder_id'" }, { "line": 803, "key": "value", "value_expression": "item.get('builderId')" }, { "line": 812, "key": "value", "value_expression": "str(entry['value'])" }, { "line": 806, "key": "type", "value_expression": "'ohs_legacy_id'" }, { "line": 806, "key": "value", "value_expression": "item.get('originalId')" }, { "line": 808, "key": "type", "value_expression": "'ohs_original_instrument_id'" }, { "line": 808, "key": "value", "value_expression": "item.get('originalInstrumentId')" }, { "line": 810, "key": "type", "value_expression": "'ohs_future_instrument_id'" }, { "line": 810, "key": "value", "value_expression": "item.get('futureInstrumentId')" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line_start": 593, "line_end": 680, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_clean_text", "_first_int", "_parse_stoplists", "_specification_compass_context", "_technical_rows" ], "literal_get_calls": [ { "line": 594, "receiver": "related", "key": "organConsoles", "expression": "related.get('organConsoles')" }, { "line": 595, "receiver": "item", "key": "manuals", "expression": "item.get('manuals')" }, { "line": 650, "receiver": "stoplist", "key": "parser", "expression": "stoplist.get('parser')" }, { "line": 661, "receiver": "compass", "key": "manual", "expression": "compass.get('manual')" }, { "line": 662, "receiver": "compass", "key": "pedal", "expression": "compass.get('pedal')" }, { "line": 614, "receiver": "consoles[0] or {}", "key": "stops", "expression": "(consoles[0] or {}).get('stops')" }, { "line": 642, "receiver": "compass", "key": "pedal", "expression": "compass.get('pedal')" }, { "line": 659, "receiver": "item", "key": "ranks", "expression": "item.get('ranks')" }, { "line": 660, "receiver": "item", "key": "pipes", "expression": "item.get('pipes')" }, { "line": 601, "receiver": "stop", "key": "division", "expression": "stop.get('division')" }, { "line": 606, "receiver": "compass", "key": "pedal", "expression": "compass.get('pedal')" }, { "line": 609, "receiver": "compass", "key": "manual", "expression": "compass.get('manual')" }, { "line": 629, "receiver": "compass", "key": "manual", "expression": "compass.get('manual')" }, { "line": 646, "receiver": "stoplist", "key": "accessories", "expression": "stoplist.get('accessories')" }, { "line": 652, "receiver": "stoplist", "key": "parser", "expression": "stoplist.get('parser')" }, { "line": 628, "receiver": "compass", "key": "manual", "expression": "compass.get('manual')" }, { "line": 640, "receiver": "compass", "key": "pedal", "expression": "compass.get('pedal')" }, { "line": 640, "receiver": "compass", "key": "pedal", "expression": "compass.get('pedal')" }, { "line": 597, "receiver": "console", "key": "manuals", "expression": "console.get('manuals')" } ], "html_queries": [], "dictionary_entries": [ { "line": 655, "key": "manual_count", "value_expression": "manual_count" }, { "line": 656, "key": "stop_count", "value_expression": "stop_count or None" }, { "line": 657, "key": "parsed_stop_count", "value_expression": "len(stops) or None" }, { "line": 658, "key": "source_stop_count", "value_expression": "source_stop_count" }, { "line": 659, "key": "rank_count", "value_expression": "self._first_int(item.get('ranks'))" }, { "line": 660, "key": "pipe_count", "value_expression": "self._first_int(item.get('pipes'))" }, { "line": 661, "key": "manual_compass", "value_expression": "compass.get('manual')" }, { "line": 662, "key": "pedal_compass", "value_expression": "compass.get('pedal')" }, { "line": 663, "key": "division_compasses", "value_expression": "compass['divisions']" }, { "line": 664, "key": "pipe_sharing_contexts", "value_expression": "compass['sharingContexts']" }, { "line": 665, "key": "manuals", "value_expression": "manuals" }, { "line": 666, "key": "pedals", "value_expression": "pedals" }, { "line": 667, "key": "werke", "value_expression": "[{'name': name, 'raw_str': name} for name in division_names]" }, { "line": 668, "key": "stops", "value_expression": "stops" }, { "line": 669, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 670, "key": "couplers", "value_expression": "couplers" }, { "line": 671, "key": "accessories", "value_expression": "accessories" }, { "line": 672, "key": "technical_rows_unstructured", "value_expression": "self._technical_rows(item, related)" }, { "line": 673, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 674, "key": "notes", "value_expression": "[]" }, { "line": 675, "key": "parser", "value_expression": "{'status': 'parsed' if stops else 'source_summary_only', 'source': 'ohs_stoplist_resources', 'resources': parser_rows}" }, { "line": 624, "key": "name", "value_expression": "f'Manual {position}'" }, { "line": 625, "key": "raw_str", "value_expression": "f'Manual {position}'" }, { "line": 626, "key": "position", "value_expression": "position" }, { "line": 676, "key": "status", "value_expression": "'parsed' if stops else 'source_summary_only'" }, { "line": 677, "key": "source", "value_expression": "'ohs_stoplist_resources'" }, { "line": 678, "key": "resources", "value_expression": "parser_rows" }, { "line": 638, "key": "name", "value_expression": "'Pedal'" }, { "line": 639, "key": "raw_str", "value_expression": "'Pedal'" }, { "line": 667, "key": "name", "value_expression": "name" }, { "line": 667, "key": "raw_str", "value_expression": "name" }, { "line": 628, "key": "note_count", "value_expression": "compass.get('manual')" }, { "line": 640, "key": "note_count", "value_expression": "compass.get('pedal')" } ], "subscript_assignments": [ { "line": 604, "value_line": 604, "key": "note_count", "target_expression": "stop['note_count']", "value_expression": "explicit" }, { "line": 605, "value_line": 605, "key": "note_count_source", "target_expression": "stop['note_count_source']", "value_expression": "'ohs_note_division_compass'" }, { "line": 613, "value_line": 613, "key": "sharing_context", "target_expression": "stop['sharing_context']", "value_expression": "compass['sharingContexts'][0]" }, { "line": 607, "value_line": 607, "key": "note_count", "target_expression": "stop['note_count']", "value_expression": "compass['pedal']" }, { "line": 608, "value_line": 608, "key": "note_count_source", "target_expression": "stop['note_count_source']", "value_expression": "'ohs_console_pedal_compass'" }, { "line": 610, "value_line": 610, "key": "note_count", "target_expression": "stop['note_count']", "value_expression": "compass['manual']" }, { "line": 611, "value_line": 611, "key": "note_count_source", "target_expression": "stop['note_count_source']", "value_expression": "'ohs_console_manual_compass'" } ] }, { "symbol": "OhsCrawler._coupler_label", "path": "aggregator/ohs_crawler.py", "line_start": 936, "line_end": 938, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 938, "receiver": "stop", "key": "length", "expression": "stop.get('length')" }, { "line": 937, "receiver": "stop", "key": "from", "expression": "stop.get('from')" }, { "line": 937, "receiver": "stop", "key": "to", "expression": "stop.get('to')" }, { "line": 938, "receiver": "stop", "key": "length", "expression": "stop.get('length')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhsCrawler._decode_stoplist_text", "path": "aggregator/ohs_crawler.py", "line_start": 404, "line_end": 410, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhsCrawler._fetch_full_record", "path": "aggregator/ohs_crawler.py", "line_start": 361, "line_end": 401, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_api_get", "_decode_stoplist_text" ], "literal_get_calls": [ { "line": 371, "receiver": "instrument", "key": "stoplists", "expression": "instrument.get('stoplists')" }, { "line": 372, "receiver": "stoplist_ref", "key": "id", "expression": "stoplist_ref.get('id')" }, { "line": 376, "receiver": "stoplist_ref", "key": "type", "expression": "stoplist_ref.get('type')" }, { "line": 379, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 387, "receiver": "response.headers", "key": "content-type", "expression": "response.headers.get('content-type')" }, { "line": 376, "receiver": "stoplist_ref", "key": "filename", "expression": "stoplist_ref.get('filename')" } ], "html_queries": [], "dictionary_entries": [ { "line": 363, "key": "instrument", "value_expression": "instrument" }, { "line": 363, "key": "related", "value_expression": "{}" }, { "line": 363, "key": "stoplists", "value_expression": "[]" }, { "line": 386, "key": "resourceUrl", "value_expression": "url" }, { "line": 387, "key": "resourceContentType", "value_expression": "response.headers.get('content-type') or 'text/plain'" }, { "line": 388, "key": "resourceSha256", "value_expression": "parsed['contentSha256']" }, { "line": 389, "key": "textContent", "value_expression": "parsed['rawText']" }, { "line": 390, "key": "stops", "value_expression": "parsed['stops']" }, { "line": 391, "key": "couplers", "value_expression": "parsed['couplers']" }, { "line": 392, "key": "accessories", "value_expression": "parsed['accessories']" }, { "line": 393, "key": "parser", "value_expression": "{key: value for key, value in parsed.items() if key not in {'rawText', 'stops', 'couplers', 'accessories', 'unresolvedLines'}}" }, { "line": 394, "key": "unresolvedLines", "value_expression": "parsed['unresolvedLines']" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._has_detail_payload", "path": "aggregator/ohs_crawler.py", "line_start": 413, "line_end": 414, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 414, "receiver": "raw_data", "key": "instrument", "expression": "raw_data.get('instrument')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhsCrawler._parse_stoplists", "path": "aggregator/ohs_crawler.py", "line_start": 682, "line_end": 733, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_clean_text", "_coupler_label", "_stop_label" ], "literal_get_calls": [ { "line": 727, "receiver": "stoplist", "key": "source", "expression": "stoplist.get('source')" }, { "line": 729, "receiver": "stoplist", "key": "description", "expression": "stoplist.get('description')" }, { "line": 731, "receiver": "stoplist", "key": "textContent", "expression": "stoplist.get('textContent')" }, { "line": 688, "receiver": "stoplist", "key": "couplers", "expression": "stoplist.get('couplers')" }, { "line": 691, "receiver": "stoplist", "key": "stops", "expression": "stoplist.get('stops')" }, { "line": 692, "receiver": "stop", "key": "from", "expression": "stop.get('from')" }, { "line": 692, "receiver": "stop", "key": "to", "expression": "stop.get('to')" }, { "line": 702, "receiver": "stop", "key": "name", "expression": "stop.get('name')" }, { "line": 702, "receiver": "stop", "key": "altName", "expression": "stop.get('altName')" }, { "line": 705, "receiver": "stop", "key": "division", "expression": "stop.get('division')" }, { "line": 710, "receiver": "stop", "key": "rank_count", "expression": "stop.get('rank_count')" }, { "line": 710, "receiver": "stop", "key": "numRanks", "expression": "stop.get('numRanks')" }, { "line": 691, "receiver": "item", "key": "order", "expression": "item.get('order')" }, { "line": 695, "receiver": "stop", "key": "from", "expression": "stop.get('from')" }, { "line": 696, "receiver": "stop", "key": "to", "expression": "stop.get('to')" }, { "line": 697, "receiver": "stop", "key": "length", "expression": "stop.get('length')" }, { "line": 709, "receiver": "stop", "key": "length", "expression": "stop.get('length')" }, { "line": 710, "receiver": "stop", "key": "rank_count", "expression": "stop.get('rank_count')" }, { "line": 711, "receiver": "stop", "key": "is_reed", "expression": "stop.get('is_reed')" }, { "line": 711, "receiver": "stop", "key": "isReed", "expression": "stop.get('isReed')" }, { "line": 712, "receiver": "stop", "key": "raw_str", "expression": "stop.get('raw_str')" }, { "line": 711, "receiver": "stop", "key": "is_reed", "expression": "stop.get('is_reed')" } ], "html_queries": [], "dictionary_entries": [ { "line": 707, "key": "name", "value_expression": "name" }, { "line": 708, "key": "division", "value_expression": "division" }, { "line": 709, "key": "length", "value_expression": "self._clean_text(stop.get('length') or '')" }, { "line": 710, "key": "rank_count", "value_expression": "stop.get('rank_count') if stop.get('rank_count') is not None else stop.get('numRanks')" }, { "line": 711, "key": "is_reed", "value_expression": "bool(stop.get('is_reed') if stop.get('is_reed') is not None else stop.get('isReed'))" }, { "line": 712, "key": "raw_str", "value_expression": "self._clean_text(stop.get('raw_str') or self._stop_label(stop))" }, { "line": 695, "key": "from", "value_expression": "stop.get('from')" }, { "line": 696, "key": "to", "value_expression": "stop.get('to')" }, { "line": 697, "key": "length", "value_expression": "stop.get('length')" }, { "line": 698, "key": "raw_str", "value_expression": "self._coupler_label(stop)" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._reference_from_instrument", "path": "aggregator/ohs_crawler.py", "line_start": 179, "line_end": 210, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_format_opus", "_format_year", "_instrument_url", "_title_for_item", "_to_text" ], "literal_get_calls": [ { "line": 183, "receiver": "item", "key": "id", "expression": "item.get('id')" }, { "line": 180, "receiver": "item", "key": "location", "expression": "item.get('location')" }, { "line": 181, "receiver": "item", "key": "builder", "expression": "item.get('builder')" }, { "line": 182, "receiver": "item", "key": "originalBuilder", "expression": "item.get('originalBuilder')" }, { "line": 184, "receiver": "raw", "key": "related", "expression": "raw.get('related')" }, { "line": 185, "receiver": "raw", "key": "stoplists", "expression": "raw.get('stoplists')" }, { "line": 188, "receiver": "item", "key": "originalId", "expression": "item.get('originalId')" }, { "line": 190, "receiver": "location", "key": "country", "expression": "location.get('country')" }, { "line": 191, "receiver": "location", "key": "state", "expression": "location.get('state')" }, { "line": 192, "receiver": "location", "key": "city", "expression": "location.get('city')" }, { "line": 193, "receiver": "location", "key": "name", "expression": "location.get('name')" }, { "line": 193, "receiver": "item", "key": "submissionLocation", "expression": "item.get('submissionLocation')" }, { "line": 194, "receiver": "item", "key": "room", "expression": "item.get('room')" }, { "line": 195, "receiver": "builder", "key": "name", "expression": "builder.get('name')" }, { "line": 195, "receiver": "item", "key": "submissionBuilder", "expression": "item.get('submissionBuilder')" }, { "line": 196, "receiver": "original_builder", "key": "name", "expression": "original_builder.get('name')" }, { "line": 196, "receiver": "item", "key": "submissionOriginalBuilder", "expression": "item.get('submissionOriginalBuilder')" }, { "line": 199, "receiver": "item", "key": "ranks", "expression": "item.get('ranks')" }, { "line": 200, "receiver": "item", "key": "pipes", "expression": "item.get('pipes')" }, { "line": 201, "receiver": "item", "key": "manuals", "expression": "item.get('manuals')" }, { "line": 184, "receiver": "raw", "key": "related", "expression": "raw.get('related')" }, { "line": 202, "receiver": "item", "key": "hasImages", "expression": "item.get('hasImages')" }, { "line": 202, "receiver": "related", "key": "images", "expression": "related.get('images')" }, { "line": 203, "receiver": "item", "key": "hasStoplists", "expression": "item.get('hasStoplists')" }, { "line": 203, "receiver": "item", "key": "stoplists", "expression": "item.get('stoplists')" }, { "line": 204, "receiver": "item", "key": "hasDocuments", "expression": "item.get('hasDocuments')" }, { "line": 204, "receiver": "related", "key": "documents", "expression": "related.get('documents')" }, { "line": 205, "receiver": "item", "key": "hasWebpages", "expression": "item.get('hasWebpages')" }, { "line": 205, "receiver": "related", "key": "webpages", "expression": "related.get('webpages')" } ], "html_queries": [], "dictionary_entries": [ { "line": 187, "key": "ohs_id", "value_expression": "ohs_id" }, { "line": 188, "key": "old_ohs_id", "value_expression": "item.get('originalId')" }, { "line": 189, "key": "title", "value_expression": "self._title_for_item(item)" }, { "line": 190, "key": "country", "value_expression": "location.get('country') or ''" }, { "line": 191, "key": "state", "value_expression": "location.get('state') or ''" }, { "line": 192, "key": "city", "value_expression": "location.get('city') or ''" }, { "line": 193, "key": "location_name", "value_expression": "location.get('name') or item.get('submissionLocation') or ''" }, { "line": 194, "key": "room", "value_expression": "item.get('room') or ''" }, { "line": 195, "key": "builder", "value_expression": "builder.get('name') or item.get('submissionBuilder') or ''" }, { "line": 196, "key": "original_builder", "value_expression": "original_builder.get('name') or item.get('submissionOriginalBuilder') or ''" }, { "line": 197, "key": "year", "value_expression": "self._format_year(item)" }, { "line": 198, "key": "opus", "value_expression": "self._format_opus(item)" }, { "line": 199, "key": "ranks", "value_expression": "self._to_text(item.get('ranks'))" }, { "line": 200, "key": "pipes", "value_expression": "self._to_text(item.get('pipes'))" }, { "line": 201, "key": "manuals", "value_expression": "self._to_text(item.get('manuals'))" }, { "line": 202, "key": "has_images", "value_expression": "int(bool(item.get('hasImages') or related.get('images')))" }, { "line": 203, "key": "has_stoplists", "value_expression": "int(bool(item.get('hasStoplists') or item.get('stoplists') or fetched_stoplists))" }, { "line": 204, "key": "has_documents", "value_expression": "int(bool(item.get('hasDocuments') or related.get('documents')))" }, { "line": 205, "key": "has_webpages", "value_expression": "int(bool(item.get('hasWebpages') or related.get('webpages')))" }, { "line": 206, "key": "url", "value_expression": "self._instrument_url(ohs_id)" }, { "line": 207, "key": "status", "value_expression": "'initiated'" }, { "line": 208, "key": "raw_data_json", "value_expression": "json.dumps(raw or item, ensure_ascii=False)" }, { "line": 209, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._resolve_identifier", "path": "aggregator/ohs_crawler.py", "line_start": 353, "line_end": 359, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_api_get", "_id_from_url" ], "literal_get_calls": [ { "line": 356, "receiver": "mapped", "key": "id", "expression": "mapped.get('id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhsCrawler._specification_compass_context", "path": "aggregator/ohs_crawler.py", "line_start": 549, "line_end": 591, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_clean_text", "_first_int" ], "literal_get_calls": [ { "line": 561, "receiver": "console", "key": "manualCompass", "expression": "console.get('manualCompass')" }, { "line": 562, "receiver": "console", "key": "pedalCompass", "expression": "console.get('pedalCompass')" }, { "line": 565, "receiver": "related", "key": "notes", "expression": "related.get('notes')" }, { "line": 557, "receiver": "related", "key": "organConsoles", "expression": "related.get('organConsoles')" }, { "line": 568, "receiver": "note", "key": "content", "expression": "note.get('content')" } ], "html_queries": [], "dictionary_entries": [ { "line": 587, "key": "manual", "value_expression": "manual_compass" }, { "line": 588, "key": "pedal", "value_expression": "pedal_compass" }, { "line": 589, "key": "divisions", "value_expression": "division_compasses" }, { "line": 590, "key": "sharingContexts", "value_expression": "list(dict.fromkeys(sharing_contexts))" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._stop_label", "path": "aggregator/ohs_crawler.py", "line_start": 932, "line_end": 933, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 933, "receiver": "stop", "key": "name", "expression": "stop.get('name')" }, { "line": 933, "receiver": "stop", "key": "length", "expression": "stop.get('length')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhsCrawler._technical_rows", "path": "aggregator/ohs_crawler.py", "line_start": 817, "line_end": 830, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [], "literal_get_calls": [ { "line": 828, "receiver": "related", "key": "organConsoles", "expression": "related.get('organConsoles')" }, { "line": 829, "receiver": "console", "key": "name", "expression": "console.get('name')" } ], "html_queries": [], "dictionary_entries": [ { "line": 829, "key": "label", "value_expression": "'Console'" }, { "line": 829, "key": "value", "value_expression": "console.get('name') or 'Console'" }, { "line": 829, "key": "raw", "value_expression": "console" }, { "line": 827, "key": "label", "value_expression": "label" }, { "line": 827, "key": "value", "value_expression": "item.get(key)" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler._title_for_item", "path": "aggregator/ohs_crawler.py", "line_start": 897, "line_end": 901, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_format_opus", "_format_year" ], "literal_get_calls": [ { "line": 898, "receiver": "item.get('builder') or {}", "key": "name", "expression": "(item.get('builder') or {}).get('name')" }, { "line": 898, "receiver": "item", "key": "submissionBuilder", "expression": "item.get('submissionBuilder')" }, { "line": 900, "receiver": "item.get('location') or {}", "key": "name", "expression": "(item.get('location') or {}).get('name')" }, { "line": 900, "receiver": "item", "key": "submissionLocation", "expression": "item.get('submissionLocation')" }, { "line": 898, "receiver": "item", "key": "builder", "expression": "item.get('builder')" }, { "line": 900, "receiver": "item", "key": "location", "expression": "item.get('location')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhsCrawler.crawl", "path": "aggregator/ohs_crawler.py", "line_start": 86, "line_end": 164, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_api_get", "_optional_flag", "_reference_from_instrument", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 87, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 91, "receiver": "kwargs", "key": "sort", "expression": "kwargs.get('sort')" }, { "line": 93, "receiver": "kwargs", "key": "builder", "expression": "kwargs.get('builder')" }, { "line": 94, "receiver": "kwargs", "key": "location", "expression": "kwargs.get('location')" }, { "line": 95, "receiver": "kwargs", "key": "opus", "expression": "kwargs.get('opus')" }, { "line": 96, "receiver": "kwargs", "key": "year", "expression": "kwargs.get('year')" }, { "line": 97, "receiver": "kwargs", "key": "city", "expression": "kwargs.get('city')" }, { "line": 98, "receiver": "kwargs", "key": "state", "expression": "kwargs.get('state')" }, { "line": 99, "receiver": "kwargs", "key": "country", "expression": "kwargs.get('country')" }, { "line": 100, "receiver": "kwargs", "key": "extant", "expression": "kwargs.get('extant')" }, { "line": 89, "receiver": "kwargs", "key": "max_pages", "expression": "kwargs.get('max_pages')" }, { "line": 90, "receiver": "kwargs", "key": "start_skip", "expression": "kwargs.get('start_skip')" }, { "line": 90, "receiver": "kwargs", "key": "skip", "expression": "kwargs.get('skip')" }, { "line": 101, "receiver": "kwargs", "key": "search_original", "expression": "kwargs.get('search_original')" }, { "line": 101, "receiver": "kwargs", "key": "searchOriginal", "expression": "kwargs.get('searchOriginal')" }, { "line": 102, "receiver": "kwargs", "key": "has_images", "expression": "kwargs.get('has_images')" }, { "line": 102, "receiver": "kwargs", "key": "hasImages", "expression": "kwargs.get('hasImages')" }, { "line": 103, "receiver": "kwargs", "key": "has_stoplists", "expression": "kwargs.get('has_stoplists')" }, { "line": 103, "receiver": "kwargs", "key": "hasStoplists", "expression": "kwargs.get('hasStoplists')" }, { "line": 104, "receiver": "kwargs", "key": "has_documents", "expression": "kwargs.get('has_documents')" }, { "line": 104, "receiver": "kwargs", "key": "hasDocuments", "expression": "kwargs.get('hasDocuments')" }, { "line": 105, "receiver": "kwargs", "key": "has_webpages", "expression": "kwargs.get('has_webpages')" }, { "line": 105, "receiver": "kwargs", "key": "hasWebpages", "expression": "kwargs.get('hasWebpages')" }, { "line": 129, "receiver": "data", "key": "instruments", "expression": "data.get('instruments')" }, { "line": 88, "receiver": "kwargs", "key": "page_size", "expression": "kwargs.get('page_size')" }, { "line": 88, "receiver": "kwargs", "key": "limit", "expression": "kwargs.get('limit')" }, { "line": 143, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.25)" } ], "html_queries": [], "dictionary_entries": [ { "line": 93, "key": "builder", "value_expression": "kwargs.get('builder')" }, { "line": 94, "key": "location", "value_expression": "kwargs.get('location')" }, { "line": 95, "key": "opus", "value_expression": "kwargs.get('opus')" }, { "line": 96, "key": "year", "value_expression": "kwargs.get('year')" }, { "line": 97, "key": "city", "value_expression": "kwargs.get('city')" }, { "line": 98, "key": "state", "value_expression": "kwargs.get('state')" }, { "line": 99, "key": "country", "value_expression": "kwargs.get('country') or 'US'" }, { "line": 100, "key": "extant", "value_expression": "self._optional_flag(kwargs.get('extant'))" }, { "line": 101, "key": "searchOriginal", "value_expression": "self._optional_flag(kwargs.get('search_original') or kwargs.get('searchOriginal'))" }, { "line": 102, "key": "hasImages", "value_expression": "self._optional_flag(kwargs.get('has_images') or kwargs.get('hasImages'))" }, { "line": 103, "key": "hasStoplists", "value_expression": "self._optional_flag(kwargs.get('has_stoplists') or kwargs.get('hasStoplists'))" }, { "line": 104, "key": "hasDocuments", "value_expression": "self._optional_flag(kwargs.get('has_documents') or kwargs.get('hasDocuments'))" }, { "line": 105, "key": "hasWebpages", "value_expression": "self._optional_flag(kwargs.get('has_webpages') or kwargs.get('hasWebpages'))" }, { "line": 106, "key": "sort", "value_expression": "sort" }, { "line": 107, "key": "limit", "value_expression": "page_size" }, { "line": 120, "key": "skip", "value_expression": "skip" } ], "subscript_assignments": [] }, { "symbol": "OhsCrawler.transform", "path": "aggregator/ohs_crawler.py", "line_start": 416, "line_end": 475, "inherited": false, "file_sha256": "b38ae2df15a04d7dd7e74b88443223c50eca797c68b802cdc7ed3da96314f81b", "self_calls": [ "_build_audio", "_build_descriptions", "_build_documents", "_build_events", "_build_images", "_build_involved", "_build_links", "_build_literature", "_build_location", "_build_other_identifiers", "_build_specifications", "_instrument_url" ], "literal_get_calls": [ { "line": 427, "receiver": "instrument", "key": "id", "expression": "instrument.get('id')" }, { "line": 449, "receiver": "payload", "key": "stoplistResourceDispositions", "expression": "payload.get('stoplistResourceDispositions')" }, { "line": 424, "receiver": "payload", "key": "instrument", "expression": "payload.get('instrument')" }, { "line": 425, "receiver": "payload", "key": "related", "expression": "payload.get('related')" }, { "line": 426, "receiver": "payload", "key": "stoplists", "expression": "payload.get('stoplists')" }, { "line": 424, "receiver": "payload", "key": "instrument", "expression": "payload.get('instrument')" }, { "line": 439, "receiver": "related", "key": "webpages", "expression": "related.get('webpages')" }, { "line": 440, "receiver": "related", "key": "references", "expression": "related.get('references')" }, { "line": 441, "receiver": "related", "key": "notes", "expression": "related.get('notes')" }, { "line": 434, "receiver": "related", "key": "images", "expression": "related.get('images')" }, { "line": 435, "receiver": "related", "key": "audio", "expression": "related.get('audio')" }, { "line": 436, "receiver": "related", "key": "documents", "expression": "related.get('documents')" } ], "html_queries": [], "dictionary_entries": [ { "line": 434, "key": "images", "value_expression": "self._build_images(related.get('images') or [])" }, { "line": 435, "key": "audio", "value_expression": "self._build_audio(related.get('audio') or [])" }, { "line": 436, "key": "documents", "value_expression": "self._build_documents(related.get('documents') or [])" }, { "line": 437, "key": "videos", "value_expression": "[]" }, { "line": 445, "key": "instrument", "value_expression": "instrument" }, { "line": 446, "key": "related", "value_expression": "related" }, { "line": 447, "key": "stoplists", "value_expression": "stoplists" }, { "line": 452, "key": "locations", "value_expression": "[location] if location else []" }, { "line": 453, "key": "involved", "value_expression": "involved" }, { "line": 454, "key": "events", "value_expression": "events" }, { "line": 455, "key": "specifications", "value_expression": "specifications" }, { "line": 456, "key": "media", "value_expression": "media" }, { "line": 457, "key": "literature", "value_expression": "literature" }, { "line": 458, "key": "links", "value_expression": "links" }, { "line": 459, "key": "descriptions", "value_expression": "descriptions" }, { "line": 460, "key": "other_identifiers", "value_expression": "self._build_other_identifiers(instrument)" }, { "line": 461, "key": "extensions", "value_expression": "{'ohs': ohs_extension}" }, { "line": 462, "key": "ohs", "value_expression": "ohs_extension" }, { "line": 466, "key": "retrieved_at", "value_expression": "retrieved_at" } ], "subscript_assignments": [ { "line": 450, "value_line": 450, "key": "stoplistResourceDispositions", "target_expression": "ohs_extension['stoplistResourceDispositions']", "value_expression": "payload['stoplistResourceDispositions']" }, { "line": 466, "value_line": 466, "key": "source_info", "target_expression": "data['source_info']", "value_expression": "{'retrieved_at': retrieved_at}" } ] } ], "selected_rule_rows": [ { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line": 656, "key": "stop_count", "value_expression": "stop_count or None" }, { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line": 655, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line": 660, "key": "pipe_count", "value_expression": "self._first_int(item.get('pipes'))" }, { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line": 659, "key": "rank_count", "value_expression": "self._first_int(item.get('ranks'))" }, { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line": 669, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "OhsCrawler._build_specifications", "path": "aggregator/ohs_crawler.py", "line": 661, "key": "manual_compass", "value_expression": "compass.get('manual')" } ], "bounded_code_passages": [ { "path": "aggregator/ohs_crawler.py", "symbol": "OhsCrawler._build_specifications", "line_start": 653, "line_end": 661, "text": " ]\n return {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count or None,\n \"parsed_stop_count\": len(stops) or None,\n \"source_stop_count\": source_stop_count,\n \"rank_count\": self._first_int(item.get(\"ranks\")),\n \"pipe_count\": self._first_int(item.get(\"pipes\")),\n \"manual_compass\": compass.get(\"manual\")," }, { "path": "aggregator/ohs_crawler.py", "symbol": "OhsCrawler._build_specifications", "line_start": 652, "line_end": 660, "text": " if isinstance(stoplist.get(\"parser\"), dict)\n ]\n return {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count or None,\n \"parsed_stop_count\": len(stops) or None,\n \"source_stop_count\": source_stop_count,\n \"rank_count\": self._first_int(item.get(\"ranks\")),\n \"pipe_count\": self._first_int(item.get(\"pipes\"))," }, { "path": "aggregator/ohs_crawler.py", "symbol": "OhsCrawler._build_specifications", "line_start": 657, "line_end": 665, "text": " \"parsed_stop_count\": len(stops) or None,\n \"source_stop_count\": source_stop_count,\n \"rank_count\": self._first_int(item.get(\"ranks\")),\n \"pipe_count\": self._first_int(item.get(\"pipes\")),\n \"manual_compass\": compass.get(\"manual\"),\n \"pedal_compass\": compass.get(\"pedal\"),\n \"division_compasses\": compass[\"divisions\"],\n \"pipe_sharing_contexts\": compass[\"sharingContexts\"],\n \"manuals\": manuals," }, { "path": "aggregator/ohs_crawler.py", "symbol": "OhsCrawler._build_specifications", "line_start": 656, "line_end": 664, "text": " \"stop_count\": stop_count or None,\n \"parsed_stop_count\": len(stops) or None,\n \"source_stop_count\": source_stop_count,\n \"rank_count\": self._first_int(item.get(\"ranks\")),\n \"pipe_count\": self._first_int(item.get(\"pipes\")),\n \"manual_compass\": compass.get(\"manual\"),\n \"pedal_compass\": compass.get(\"pedal\"),\n \"division_compasses\": compass[\"divisions\"],\n \"pipe_sharing_contexts\": compass[\"sharingContexts\"]," }, { "path": "aggregator/ohs_crawler.py", "symbol": "OhsCrawler._build_specifications", "line_start": 666, "line_end": 674, "text": " \"pedals\": pedals,\n \"werke\": [{\"name\": name, \"raw_str\": name} for name in division_names],\n \"stops\": stops,\n \"stops_by_division\": stops_by_division,\n \"couplers\": couplers,\n \"accessories\": accessories,\n \"technical_rows_unstructured\": self._technical_rows(item, related),\n \"raw_spec\": raw_spec,\n \"notes\": []," }, { "path": "aggregator/ohs_crawler.py", "symbol": "OhsCrawler._build_specifications", "line_start": 658, "line_end": 666, "text": " \"source_stop_count\": source_stop_count,\n \"rank_count\": self._first_int(item.get(\"ranks\")),\n \"pipe_count\": self._first_int(item.get(\"pipes\")),\n \"manual_compass\": compass.get(\"manual\"),\n \"pedal_compass\": compass.get(\"pedal\"),\n \"division_compasses\": compass[\"divisions\"],\n \"pipe_sharing_contexts\": compass[\"sharingContexts\"],\n \"manuals\": manuals,\n \"pedals\": pedals," } ] } }, { "catalogue_id": "S027", "source_key": "ohta", "name": "OHTA Organs of Australia", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Organ Historical Trust of Australia", "scope_description": "Australian pipe-organ heritage", "documentary_unit": "Gazetteer entry and associated documentation", "source_description": "OHTA combines gazetteers with conservation guidance, builder documentation and a separate John Stiller documentation project dated 1978-1986. It also lists relocated and redundant organs.", "source_description_references": [ "ohta", "ohta_rights", "record_ohta" ], "interpretation": "The historical survey and later gazetteer entries are distinct documentary layers. Relocation is especially important when matching an organ by its present address. Page authorship and the date of the underlying inspection should be retained where supplied.", "rights_observation": "The published policy limits use to personal, educational and non-commercial purposes and requires permission for reproduction.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "ohta", "ohta_rights", "record_ohta" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_ohta", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Gazetteer article with footnoted historical essay and specification", "repeated_information": "Earlier and present organs occupy separate sections; stops are grouped by manual and pedal.", "date_and_state_distinctions": "The selected article separates a removed predecessor from the present instrument and records its later destination.", "interpretive_limit": "One venue article can describe multiple organs. Count instrument states and relocations separately from page count.", "observations": [ "Church and locality are identified in the article.", "Earlier and present builders and an opus are named.", "Construction, relocation and replacement are dated.", "Present instrument has a division-based stoplist with pitches.", "Manual/pedal note counts and division labels are stated.", "Mechanical key and stop action is stated.", "Equal temperament is stated.", "Case material, key coverings and pedalboard design are described.", "Removal and later installation of the predecessor are documented.", "Dated and attributed church, organ and console photographs.", "Numbered notes cite publications and on-site observations.", "Links connect the relocated instrument and related venues.", "Named author, observation dates and page-update date are supplied." ], "uninspected_or_unfilled_material": [], "narrative": "The Brisbane Lutheran article treats an earlier, relocated organ separately from the present instrument. Its specification includes division grouping, compass, action and console details. Footnotes and photographic credits identify documentary sources and observations. This is a venue history containing more than one instrument, rather than one timeless specification." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Index links lead to acquired organ articles; the adapter applies the static-page family and source-specific location/title handling.", "transformation": "The common transform reads page text, metadata, specification candidates and images. The OHTA override checks whether a reference year is actually accompanied by the word opus and removes matching event/participant projections with an explicit warning. Its location rule interprets a comma-separated title as building followed by town.", "limitations": "An article may document predecessors and successors. The opus safeguard addresses one known ambiguity, not every misleading number or historical date. The generic first-year fallback and construction projection still need checking against the article when no more explicit metadata is available." }, "implementation": { "class": "OhtaCrawler", "class_line": 867, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OhtaCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "ohta", "path": "aggregator/static_html_crawler.py", "line": 868, "class": "OhtaCrawler" }, "id_column": { "value": "ohta_id", "path": "aggregator/static_html_crawler.py", "line": 869, "class": "OhtaCrawler" }, "default_country": { "value": "Australia", "path": "aggregator/static_html_crawler.py", "line": 872, "class": "OhtaCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://ohta.org.au/organs/organs/QueenslandOrganHistory.html", "https://ohta.org.au/gazetteers/gazetteer-victorian-pipe-organs/", "https://ohta.org.au/gazetteers/gazetteer-queensland-pipe-organs/", "https://ohta.org.au/gazetteers/gazetteer-tasmanian-pipe-organs/", "https://ohta.org.au/gazetteers/gazetteer-nsw-pipe-organs/" ], "path": "aggregator/static_html_crawler.py", "line": 873, "class": "OhtaCrawler" }, "detail_url_patterns": { "value": [ "/organs/" ], "path": "aggregator/static_html_crawler.py", "line": 880, "class": "OhtaCrawler" }, "exclude_url_patterns": { "value": [ "queenslandorganhistory.html#", "organs-of-australia", "historical-overviews", "pipe-organ-documentation-project", ".pdf" ], "path": "aggregator/static_html_crawler.py", "line": 881, "class": "OhtaCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://ohta.org.au", "path": "aggregator/static_html_crawler.py", "line": 871, "class": "OhtaCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhtaCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 889, "line_end": 893, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhtaCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 905, "line_end": 909, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 202, "line_end": 211, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 205, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 895, "line_end": 903, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 897, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 900, "receiver": "record", "key": "events", "expression": "record.get('events', [])" }, { "line": 901, "receiver": "record", "key": "involved", "expression": "record.get('involved', [])" }, { "line": 898, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 898, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 900, "receiver": "event", "key": "dates", "expression": "event.get('dates', [])" }, { "line": 900, "receiver": "event", "key": "date", "expression": "event.get('date')" }, { "line": 901, "receiver": "item", "key": "dates", "expression": "item.get('dates', [])" }, { "line": 901, "receiver": "item", "key": "year", "expression": "item.get('year')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 900, "value_line": 900, "key": "events", "target_expression": "record['events']", "value_expression": "[event for event in record.get('events', []) if year not in event.get('dates', []) and event.get('date') != year]" }, { "line": 901, "value_line": 901, "key": "involved", "target_expression": "record['involved']", "value_expression": "[item for item in record.get('involved', []) if year not in item.get('dates', []) and item.get('year') != year]" } ] } ], "selected_rule_rows": [ { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "year", "value_expression": "raw_data.get('year')", "line": 897, "kind": "input read" }, { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "events", "value_expression": "record.get('events', [])", "line": 900, "kind": "input read" }, { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "involved", "value_expression": "record.get('involved', [])", "line": 901, "kind": "input read" }, { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "html", "value_expression": "raw_data.get('html')", "line": 898, "kind": "input read" }, { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "raw_html", "value_expression": "raw_data.get('raw_html')", "line": 898, "kind": "input read" }, { "symbol": "OhtaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "dates", "value_expression": "event.get('dates', [])", "line": 900, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OhtaCrawler.transform", "line_start": 895, "line_end": 902, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n record = super().transform(raw_data)\n year = self._clean_text(raw_data.get(\"year\"))\n source_text = BeautifulSoup(raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\", \"html.parser\").get_text(\" \", strip=True)\n if year and re.search(rf\"\\bopus\\s*(?:no\\.?\\s*)?{re.escape(year)}\\b\", source_text, flags=re.I):\n record[\"events\"] = [event for event in record.get(\"events\", []) if year not in event.get(\"dates\", []) and event.get(\"date\") != year]\n record[\"involved\"] = [item for item in record.get(\"involved\", []) if year not in item.get(\"dates\", []) and item.get(\"year\") != year]\n record.setdefault(\"quality\", {}).setdefault(\"warnings\", []).append(\"opus_number_not_mapped_as_year\")" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OhtaCrawler.transform", "line_start": 897, "line_end": 903, "text": " year = self._clean_text(raw_data.get(\"year\"))\n source_text = BeautifulSoup(raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\", \"html.parser\").get_text(\" \", strip=True)\n if year and re.search(rf\"\\bopus\\s*(?:no\\.?\\s*)?{re.escape(year)}\\b\", source_text, flags=re.I):\n record[\"events\"] = [event for event in record.get(\"events\", []) if year not in event.get(\"dates\", []) and event.get(\"date\") != year]\n record[\"involved\"] = [item for item in record.get(\"involved\", []) if year not in item.get(\"dates\", []) and item.get(\"year\") != year]\n record.setdefault(\"quality\", {}).setdefault(\"warnings\", []).append(\"opus_number_not_mapped_as_year\")\n return record" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OhtaCrawler.transform", "line_start": 898, "line_end": 903, "text": " source_text = BeautifulSoup(raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\", \"html.parser\").get_text(\" \", strip=True)\n if year and re.search(rf\"\\bopus\\s*(?:no\\.?\\s*)?{re.escape(year)}\\b\", source_text, flags=re.I):\n record[\"events\"] = [event for event in record.get(\"events\", []) if year not in event.get(\"dates\", []) and event.get(\"date\") != year]\n record[\"involved\"] = [item for item in record.get(\"involved\", []) if year not in item.get(\"dates\", []) and item.get(\"year\") != year]\n record.setdefault(\"quality\", {}).setdefault(\"warnings\", []).append(\"opus_number_not_mapped_as_year\")\n return record" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OhtaCrawler.transform", "line_start": 895, "line_end": 903, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n record = super().transform(raw_data)\n year = self._clean_text(raw_data.get(\"year\"))\n source_text = BeautifulSoup(raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\", \"html.parser\").get_text(\" \", strip=True)\n if year and re.search(rf\"\\bopus\\s*(?:no\\.?\\s*)?{re.escape(year)}\\b\", source_text, flags=re.I):\n record[\"events\"] = [event for event in record.get(\"events\", []) if year not in event.get(\"dates\", []) and event.get(\"date\") != year]\n record[\"involved\"] = [item for item in record.get(\"involved\", []) if year not in item.get(\"dates\", []) and item.get(\"year\") != year]\n record.setdefault(\"quality\", {}).setdefault(\"warnings\", []).append(\"opus_number_not_mapped_as_year\")\n return record" } ] } }, { "catalogue_id": "S028", "source_key": "organindex", "name": "Organ Index", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Organ Index contributors", "scope_description": "Organ descriptions in a collaborative wiki", "documentary_unit": "MediaWiki article and revision", "source_description": "Aggregator includes this organ source in its selection frame. The public entry point returned HTTP 403 during the review; its current articles, statistics and licence were not verified.", "source_description_references": [ "organindex", "selection_scope", "record_organindex" ], "interpretation": "A revision identifier is preferable to an unversioned article URL. Article titles can change, and shared text or images may originate elsewhere. The failed request is an access observation, not evidence that the wiki has ceased operation.", "rights_observation": "Current wiki and image licence terms remain unverified in this review.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "organindex", "selection_scope", "record_organindex" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_organindex", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "access_finding": "The configured entry URL returned HTTP 403. No article was inspected." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The acquisition route follows wiki categories, category continuations and linked detail pages. The code also provides import routes for already captured category and detail HTML. Page title and category membership contribute to discovery; they are not organ identities across time.", "transformation": "The detail transform reads a metadata table through a label map, constructs location and builder information, retains history text, and parses division columns below disposition headings. Images, literature, external links and categories have separate extractors. The disposition routine selects the next table after a matching heading and exits after the first successful parse.", "limitations": "That first-success rule can omit additional specifications from the structured stoplist even when the page contains them. The table routine expects matching header and value cells in its first two direct rows. Other layouts require separate inspection. A wiki page can be richer than the fields populated by this parser; revision history is not established by a title or one captured page." }, "implementation": { "class": "OrganIndexCrawler", "class_line": 25, "path": "aggregator/organindex_crawler.py", "inheritance": [ "OrganIndexCrawler", "BaseCrawler" ], "configuration": { "LABEL_MAP": { "value": { "orgelbauer": "builder", "baujahr": "year", "geschichte der orgel": "history", "umbauten": "rebuilds", "temperatur (stimmung)": "temperament", "windladen": "windchest", "spieltraktur": "key_action", "registertraktur": "stop_action", "registeranzahl": "stop_count", "manuale": "manuals", "pedal": "pedal", "spielhilfen, koppeln": "couplers" }, "path": "aggregator/organindex_crawler.py", "line": 30, "class": "OrganIndexCrawler" } }, "methods": [ { "symbol": "OrganIndexCrawler._build_location", "path": "aggregator/organindex_crawler.py", "line_start": 601, "line_end": 619, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_parse_address", "_split_title" ], "literal_get_calls": [ { "line": 609, "receiver": "summary", "key": "address", "expression": "summary.get('address')" }, { "line": 613, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 614, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 615, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 616, "receiver": "summary", "key": "building", "expression": "summary.get('building')" }, { "line": 616, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [], "dictionary_entries": [ { "line": 612, "key": "status", "value_expression": "'current_location'" }, { "line": 613, "key": "country", "value_expression": "country or raw_data.get('country') or 'Germany'" }, { "line": 614, "key": "region", "value_expression": "region or raw_data.get('region') or ''" }, { "line": 615, "key": "town", "value_expression": "parsed_town or raw_data.get('town') or town" }, { "line": 616, "key": "building", "value_expression": "summary.get('building') or raw_data.get('building') or title_building" }, { "line": 617, "key": "address", "value_expression": "address" }, { "line": 618, "key": "raw_str", "value_expression": "address or title" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._clean_label", "path": "aggregator/organindex_crawler.py", "line_start": 840, "line_end": 841, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._clean_stop_text", "path": "aggregator/organindex_crawler.py", "line_start": 851, "line_end": 857, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_categories", "path": "aggregator/organindex_crawler.py", "line_start": 783, "line_end": 789, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 785, "expression": "soup.select('#catlinks a, .mw-normal-catlinks a')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_descriptions", "path": "aggregator/organindex_crawler.py", "line_start": 653, "line_end": 657, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [ { "line": 655, "receiver": "metadata", "key": "history", "expression": "metadata.get('history')" } ], "html_queries": [], "dictionary_entries": [ { "line": 656, "key": "type", "value_expression": "'history'" }, { "line": 656, "key": "text", "value_expression": "metadata['history']" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_dispositions", "path": "aggregator/organindex_crawler.py", "line_start": 693, "line_end": 708, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text", "_parse_disposition_table" ], "literal_get_calls": [], "html_queries": [ { "line": 696, "expression": "soup.find_all(['h2', 'h3', 'h4'])" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_events", "path": "aggregator/organindex_crawler.py", "line_start": 645, "line_end": 651, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [ { "line": 647, "receiver": "metadata", "key": "year", "expression": "metadata.get('year')" }, { "line": 649, "receiver": "metadata", "key": "rebuilds", "expression": "metadata.get('rebuilds')" }, { "line": 648, "receiver": "metadata", "key": "builder", "expression": "metadata.get('builder', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 648, "key": "type", "value_expression": "'construction'" }, { "line": 648, "key": "date", "value_expression": "metadata['year']" }, { "line": 648, "key": "description", "value_expression": "metadata.get('builder', '')" }, { "line": 650, "key": "type", "value_expression": "'rebuild'" }, { "line": 650, "key": "date", "value_expression": "metadata['rebuilds']" }, { "line": 650, "key": "description", "value_expression": "metadata['rebuilds']" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_images", "path": "aggregator/organindex_crawler.py", "line_start": 730, "line_end": 746, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text", "_looks_like_map_or_logo" ], "literal_get_calls": [ { "line": 733, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 736, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 736, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 732, "expression": "soup.select('.thumb img, figure img, a.image img')" } ], "dictionary_entries": [ { "line": 741, "key": "link", "value_expression": "urljoin(self.base_url, src)" }, { "line": 742, "key": "caption", "value_expression": "alt" }, { "line": 743, "key": "source", "value_expression": "self.source_name" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_involved", "path": "aggregator/organindex_crawler.py", "line_start": 632, "line_end": 643, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [ { "line": 633, "receiver": "metadata", "key": "builder", "expression": "metadata.get('builder')" }, { "line": 640, "receiver": "metadata", "key": "year", "expression": "metadata.get('year')" }, { "line": 640, "receiver": "metadata", "key": "year", "expression": "metadata.get('year')" } ], "html_queries": [], "dictionary_entries": [ { "line": 638, "key": "name", "value_expression": "builder" }, { "line": 639, "key": "activities", "value_expression": "['builder']" }, { "line": 640, "key": "dates", "value_expression": "[metadata.get('year')] if metadata.get('year') else []" }, { "line": 641, "key": "raw_str", "value_expression": "builder" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_links", "path": "aggregator/organindex_crawler.py", "line_start": 753, "line_end": 764, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 756, "expression": "content.find_all('a', href=True)" }, { "line": 755, "expression": "soup.select_one('#mw-content-text')" } ], "dictionary_entries": [ { "line": 763, "key": "url", "value_expression": "full_url" }, { "line": 763, "key": "label", "value_expression": "self._clean_text(link.get_text(' ', strip=True))" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_literature", "path": "aggregator/organindex_crawler.py", "line_start": 766, "line_end": 781, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 768, "expression": "soup.find_all(['h2', 'h3'])" } ], "dictionary_entries": [ { "line": 779, "key": "raw_str", "value_expression": "'\\n'.join(pieces)" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_metadata_table", "path": "aggregator/organindex_crawler.py", "line_start": 581, "line_end": 599, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_label", "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 583, "expression": "soup.find_all('table')" }, { "line": 586, "expression": "table.find_all('tr', recursive=False)" }, { "line": 587, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_specifications", "path": "aggregator/organindex_crawler.py", "line_start": 659, "line_end": 691, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_extract_dispositions", "_first_int" ], "literal_get_calls": [ { "line": 664, "receiver": "metadata", "key": "stop_count", "expression": "metadata.get('stop_count')" }, { "line": 663, "receiver": "metadata", "key": "manuals", "expression": "metadata.get('manuals')" }, { "line": 678, "receiver": "metadata", "key": "couplers", "expression": "metadata.get('couplers')" } ], "html_queries": [], "dictionary_entries": [ { "line": 668, "key": "manual_count", "value_expression": "manual_count" }, { "line": 669, "key": "stop_count", "value_expression": "stop_count" }, { "line": 670, "key": "manuals", "value_expression": "[{'name': name, 'raw_str': name} for name in manual_names]" }, { "line": 671, "key": "pedals", "value_expression": "[{'name': name, 'raw_str': name} for name in pedal_names]" }, { "line": 672, "key": "stops_by_division", "value_expression": "divisions" }, { "line": 673, "key": "stops", "value_expression": "[{'name': stop, 'division': division, 'raw_str': stop} for division, stops in divisions.items() for stop in stops]" }, { "line": 678, "key": "couplers", "value_expression": "[{'raw_str': metadata['couplers']}] if metadata.get('couplers') else []" }, { "line": 679, "key": "technical_rows_unstructured", "value_expression": "[{'label': label, 'value': metadata[key]} for label, key in (('Windladen', 'windchest'), ('Spieltraktur', 'key_action'), ('Registertraktur', 'stop_action'), ('Temperatur', 'temperament')) if metadata.get(key)]" }, { "line": 689, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 690, "key": "notes", "value_expression": "[]" }, { "line": 670, "key": "name", "value_expression": "name" }, { "line": 670, "key": "raw_str", "value_expression": "name" }, { "line": 671, "key": "name", "value_expression": "name" }, { "line": 671, "key": "raw_str", "value_expression": "name" }, { "line": 674, "key": "name", "value_expression": "stop" }, { "line": 674, "key": "division", "value_expression": "division" }, { "line": 674, "key": "raw_str", "value_expression": "stop" }, { "line": 680, "key": "label", "value_expression": "label" }, { "line": 680, "key": "value", "value_expression": "metadata[key]" }, { "line": 678, "key": "raw_str", "value_expression": "metadata['couplers']" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._extract_summary", "path": "aggregator/organindex_crawler.py", "line_start": 569, "line_end": 579, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 578, "value_line": 578, "key": "building", "target_expression": "summary['building']", "value_expression": "building" } ] }, { "symbol": "OrganIndexCrawler._fetch_html", "path": "aggregator/organindex_crawler.py", "line_start": 93, "line_end": 103, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_is_challenge_page" ], "literal_get_calls": [ { "line": 94, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._page_title", "path": "aggregator/organindex_crawler.py", "line_start": 565, "line_end": 567, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 566, "expression": "soup.select_one('#firstHeading, h1')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._parse_address", "path": "aggregator/organindex_crawler.py", "line_start": 622, "line_end": 630, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._parse_category_page", "path": "aggregator/organindex_crawler.py", "line_start": 277, "line_end": 313, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_split_title", "_title_from_link" ], "literal_get_calls": [ { "line": 309, "receiver": "parse_qs(urlparse(link['href']).query)", "key": "pagefrom", "expression": "parse_qs(urlparse(link['href']).query).get('pagefrom', [None])" } ], "html_queries": [ { "line": 279, "expression": "soup.select_one('#mw-pages')" }, { "line": 284, "expression": "pages_container.select('a[href]')" }, { "line": 306, "expression": "pages_container.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 293, "key": "organindex_id", "value_expression": "title" }, { "line": 294, "key": "title", "value_expression": "title" }, { "line": 295, "key": "town", "value_expression": "town" }, { "line": 296, "key": "building", "value_expression": "building" }, { "line": 297, "key": "region", "value_expression": "category.replace('Kategorie:', '')" }, { "line": 298, "key": "country", "value_expression": "'Germany'" }, { "line": 299, "key": "url", "value_expression": "urljoin(self.base_url, link['href'])" }, { "line": 300, "key": "status", "value_expression": "'initiated'" }, { "line": 301, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._parse_disposition_table", "path": "aggregator/organindex_crawler.py", "line_start": 710, "line_end": 728, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_stop_text", "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 711, "expression": "table.find_all('tr', recursive=False)" }, { "line": 715, "expression": "rows[1].find_all(['th', 'td'], recursive=False)" }, { "line": 714, "expression": "rows[0].find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._split_title", "path": "aggregator/organindex_crawler.py", "line_start": 378, "line_end": 385, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler._title_from_link", "path": "aggregator/organindex_crawler.py", "line_start": 370, "line_end": 375, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 371, "receiver": "link", "key": "title", "expression": "link.get('title')" }, { "line": 372, "receiver": "parse_qs(urlparse(link.get('href', '')).query)", "key": "title", "expression": "parse_qs(urlparse(link.get('href', '')).query).get('title', [None])" }, { "line": 372, "receiver": "link", "key": "href", "expression": "link.get('href', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler.crawl", "path": "aggregator/organindex_crawler.py", "line_start": 149, "line_end": 226, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_as_bool", "_category_url", "_clean_text", "_fetch_html", "_load_resume_cursor", "_parse_category_page", "_save_reference", "_save_resume_cursor", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 153, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 150, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 151, "receiver": "kwargs", "key": "category", "expression": "kwargs.get('category')" }, { "line": 155, "receiver": "kwargs", "key": "resume", "expression": "kwargs.get('resume', True)" }, { "line": 152, "receiver": "kwargs", "key": "max_pages", "expression": "kwargs.get('max_pages')" }, { "line": 156, "receiver": "kwargs", "key": "page_from", "expression": "kwargs.get('page_from')" }, { "line": 156, "receiver": "kwargs", "key": "start_after", "expression": "kwargs.get('start_after')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler.import_detail_html", "path": "aggregator/organindex_crawler.py", "line_start": 331, "line_end": 368, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_clean_text", "_is_challenge_page", "_page_title", "_page_url", "_save_reference", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 348, "key": "organindex_id", "value_expression": "title" }, { "line": 349, "key": "title", "value_expression": "title" }, { "line": 350, "key": "town", "value_expression": "town" }, { "line": 351, "key": "building", "value_expression": "building" }, { "line": 352, "key": "region", "value_expression": "category.replace('Kategorie:', '')" }, { "line": 353, "key": "country", "value_expression": "'Germany'" }, { "line": 354, "key": "url", "value_expression": "url or self._page_url(title)" }, { "line": 355, "key": "status", "value_expression": "'scraped'" }, { "line": 356, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" }, { "line": 368, "key": "kind", "value_expression": "'detail'" }, { "line": 368, "key": "identifier", "value_expression": "title" }, { "line": 368, "key": "url", "value_expression": "ref['url']" } ], "subscript_assignments": [] }, { "symbol": "OrganIndexCrawler.transform", "path": "aggregator/organindex_crawler.py", "line_start": 518, "line_end": 563, "inherited": false, "file_sha256": "944cdc3083afff17d453215724a32deb5ea874f18cdb7063c7d88a68f31c6938", "self_calls": [ "_build_location", "_clean_text", "_extract_categories", "_extract_descriptions", "_extract_events", "_extract_images", "_extract_involved", "_extract_links", "_extract_literature", "_extract_metadata_table", "_extract_specifications", "_extract_summary", "_is_challenge_page", "_page_title" ], "literal_get_calls": [ { "line": 519, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "organindex_id", "expression": "raw_data.get('organindex_id', '')" }, { "line": 560, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 559, "receiver": "raw_data", "key": "organindex_id", "expression": "raw_data.get('organindex_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 529, "key": "images", "value_expression": "self._extract_images(soup)" }, { "line": 536, "key": "locations", "value_expression": "[location] if location else []" }, { "line": 537, "key": "involved", "value_expression": "involved" }, { "line": 538, "key": "events", "value_expression": "self._extract_events(metadata)" }, { "line": 539, "key": "specifications", "value_expression": "specifications" }, { "line": 540, "key": "media", "value_expression": "media" }, { "line": 541, "key": "literature", "value_expression": "literature" }, { "line": 542, "key": "links", "value_expression": "links" }, { "line": 543, "key": "descriptions", "value_expression": "descriptions" }, { "line": 544, "key": "other_identifiers", "value_expression": "[{'type': 'organindex_title', 'value': title}]" }, { "line": 547, "key": "extensions", "value_expression": "{'organindex': {'title': title, 'metadata': metadata, 'summary': summary, 'categories': self._extract_categories(soup)}}" }, { "line": 548, "key": "organindex", "value_expression": "{'title': title, 'metadata': metadata, 'summary': summary, 'categories': self._extract_categories(soup)}" }, { "line": 545, "key": "type", "value_expression": "'organindex_title'" }, { "line": 545, "key": "value", "value_expression": "title" }, { "line": 549, "key": "title", "value_expression": "title" }, { "line": 550, "key": "metadata", "value_expression": "metadata" }, { "line": 551, "key": "summary", "value_expression": "summary" }, { "line": 552, "key": "categories", "value_expression": "self._extract_categories(soup)" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "OrganIndexCrawler._extract_specifications", "path": "aggregator/organindex_crawler.py", "line": 669, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "OrganIndexCrawler._extract_specifications", "path": "aggregator/organindex_crawler.py", "line": 668, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "OrganIndexCrawler._extract_specifications", "path": "aggregator/organindex_crawler.py", "line": 672, "key": "stops_by_division", "value_expression": "divisions" }, { "symbol": "OrganIndexCrawler._extract_involved", "path": "aggregator/organindex_crawler.py", "line": 640, "key": "dates", "value_expression": "[metadata.get('year')] if metadata.get('year') else []" }, { "symbol": "OrganIndexCrawler._parse_category_page", "path": "aggregator/organindex_crawler.py", "line": 298, "key": "country", "value_expression": "'Germany'" }, { "symbol": "OrganIndexCrawler._build_location", "path": "aggregator/organindex_crawler.py", "line": 618, "key": "raw_str", "value_expression": "address or title" } ], "bounded_code_passages": [ { "path": "aggregator/organindex_crawler.py", "symbol": "OrganIndexCrawler._extract_specifications", "line_start": 666, "line_end": 674, "text": " stop_count = sum(len(stops) for stops in divisions.values()) or None\n return {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"manuals\": [{\"name\": name, \"raw_str\": name} for name in manual_names],\n \"pedals\": [{\"name\": name, \"raw_str\": name} for name in pedal_names],\n \"stops_by_division\": divisions,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}" }, { "path": "aggregator/organindex_crawler.py", "symbol": "OrganIndexCrawler._extract_specifications", "line_start": 665, "line_end": 673, "text": " if stop_count is None:\n stop_count = sum(len(stops) for stops in divisions.values()) or None\n return {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"manuals\": [{\"name\": name, \"raw_str\": name} for name in manual_names],\n \"pedals\": [{\"name\": name, \"raw_str\": name} for name in pedal_names],\n \"stops_by_division\": divisions,\n \"stops\": [" }, { "path": "aggregator/organindex_crawler.py", "symbol": "OrganIndexCrawler._extract_specifications", "line_start": 669, "line_end": 677, "text": " \"stop_count\": stop_count,\n \"manuals\": [{\"name\": name, \"raw_str\": name} for name in manual_names],\n \"pedals\": [{\"name\": name, \"raw_str\": name} for name in pedal_names],\n \"stops_by_division\": divisions,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in divisions.items()\n for stop in stops\n ]," }, { "path": "aggregator/organindex_crawler.py", "symbol": "OrganIndexCrawler._extract_involved", "line_start": 637, "line_end": 643, "text": " {\n \"name\": builder,\n \"activities\": [\"builder\"],\n \"dates\": [metadata.get(\"year\")] if metadata.get(\"year\") else [],\n \"raw_str\": builder,\n }\n ]" }, { "path": "aggregator/organindex_crawler.py", "symbol": "OrganIndexCrawler._parse_category_page", "line_start": 295, "line_end": 303, "text": " \"town\": town,\n \"building\": building,\n \"region\": category.replace(\"Kategorie:\", \"\"),\n \"country\": \"Germany\",\n \"url\": urljoin(self.base_url, link[\"href\"]),\n \"status\": \"initiated\",\n \"scraped_at\": datetime.now().isoformat(),\n }\n )" }, { "path": "aggregator/organindex_crawler.py", "symbol": "OrganIndexCrawler._build_location", "line_start": 615, "line_end": 619, "text": " \"town\": parsed_town or raw_data.get(\"town\") or town,\n \"building\": summary.get(\"building\") or raw_data.get(\"building\") or title_building,\n \"address\": address,\n \"raw_str\": address or title,\n }" } ] } }, { "catalogue_id": "S029", "source_key": "orgbase", "name": "Orgbase", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Orgbase / Organ Database", "scope_description": "International; multilingual interface", "documentary_unit": "Organ description and specification", "source_description": "Orgbase describes instruments worldwide and provides searches by builder, place, building, construction year and text. Its information page dates the database's beginning to October 1999. DBIS, the library database directory, reports more than 75,000 records; the consulted entry supplies no date for that quantity. The home page explicitly notes that not all descriptive information is available in every language.", "source_description_references": [ "orgbase", "orgbase_about", "orgbase_dbis", "record_orgbase" ], "interpretation": "The DBIS figure establishes a reported scale, not a verified current total or a count of distinct surviving instruments. Language-specific versions may be incomplete relative to one another. Searches and comparisons should preserve the original language and source identifier. The information page's first-person account supplies a history but no named author in the retrieved text.", "rights_observation": "No general open licence was established from the home and information pages.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgbase", "orgbase_about", "orgbase_dbis", "record_orgbase" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgbase", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Searchable database with parameterised description records and technical sections", "repeated_information": "Division stop counts precede prose-form division specifications, couplers and bibliography.", "date_and_state_distinctions": "Builder date and documentary literature dates are distinct. Language versions may differ in completeness.", "interpretive_limit": "Comma-separated division text includes pitches, mixture ranks, borrowing and compass information. It is not a uniform row-per-stop table.", "observations": [ "Country, region, city and named concert venue are populated.", "Named builder and construction year.", "Builder year and opening account provide dated instrument history.", "Stops are listed as comma-separated entries within divisions.", "Division labels and manual/pedal compasses are supplied.", "Separate key action, stop action and windchest fields.", "Pitch and temperament are separately populated.", "Wind pressure is stated with a unit.", "Couplers and a combination-system accessory are specified.", "Instrument photograph with named builder credit.", "A bibliography gives journal titles, issue details and authors.", "The venue website and source description identifier are present.", "The image supplier is credited; bibliography attribution is item-specific." ], "uninspected_or_unfilled_material": [], "narrative": "Record 2000011, for the Kazan concert organ, combines labelled technical attributes with division-based, comma-separated specifications. It distinguishes key action, stop action, windchest, pitch, temperament and wind pressure, and supplies compass information, playing aids and bibliography. Its stoplist is not a uniform table with one row per stop. The record identifier and language parameter belong in a reproducible citation." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Country-list discovery and detail-page transformation are separate operations. A list row supplies an identifier and location, builder and manual-summary context; the HTML detail supplies the documentary sections. The offline transformer accepts already acquired evidence, so acquisition and interpretation can be studied separately.", "transformation": "The transform partitions the page into identification and location, activity tables, technical information, alternative specifications, footer references and images. Location histories preserve source-row positions. The alternative-specification branch collects labelled groups and their divisions; the compound-stop branch is distinct from ordinary stop parsing. Footer literature is merged with table literature rather than replacing it. A missing detail location can be filled from the listing context. These fallbacks and merges are part of the interpretation and must remain distinguishable from literal source statements. Technical labels are mapped individually: total stops, key action, stop action, windchests, temperament, pitch, wind pressure and division totals. Unrecognised technical rows are retained. At stop level the parser separates date strings, parenthetical notes, mixture-rank wording and pipe lengths from a cleaned name. The first detected pipe length supplies a MIDI conversion, and the resulting number is used to calculate a sounding transposition.", "limitations": "A source record can therefore carry several places and specifications. Do not assign every stop to one present-day configuration. The location parser skips rows without a recognised year expression; its retention rule is narrower than all rows containing meaningful prose. A parsed division, stop or activity is not an independently identified physical entity. The presence of alternative-specification code establishes implemented handling, not that every historical layout is correctly parsed. The B/D branch supplies fixed bass and discant MIDI intervals, [36, 59] and [60, 96]. Those intervals are parser defaults, not measurements of that organ's split point or compass. A No stops present input is converted to an unavailable/unknown placeholder. The cleaned stop name, derived pitch representation and preserved wording consequently have different evidential status." }, "implementation": { "class": "OrgbaseCrawler", "class_line": 96, "path": "aggregator/orgbase_crawler.py", "inheritance": [ "OrgbaseCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "OrgbaseCrawler._address_line_after_label", "path": "aggregator/orgbase_crawler.py", "line_start": 1193, "line_end": 1203, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._apply_country_to_location", "path": "aggregator/orgbase_crawler.py", "line_start": 1092, "line_end": 1097, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_normalize_country" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 1095, "value_line": 1095, "key": "country", "target_expression": "location['country']", "value_expression": "country_final" }, { "line": 1097, "value_line": 1097, "key": "country_original", "target_expression": "location['country_original']", "value_expression": "country_original" } ] }, { "symbol": "OrgbaseCrawler._apply_reference_manual_summary", "path": "aggregator/orgbase_crawler.py", "line_start": 728, "line_end": 745, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_roman_to_int" ], "literal_get_calls": [ { "line": 736, "receiver": "raw_data", "key": "manuals", "expression": "raw_data.get('manuals')" }, { "line": 736, "receiver": "raw_data", "key": "manuals_raw", "expression": "raw_data.get('manuals_raw')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 731, "value_line": 731, "key": "reference_context", "target_expression": "extension['reference_context']", "value_expression": "{key: raw_data.get(key) for key in ('orgbase_id', 'country', 'town', 'building', 'builder', 'manuals', 'url') if raw_data.get(key) not in (None, '')}" }, { "line": 744, "value_line": 744, "key": "manual_summary_raw", "target_expression": "specifications['manual_summary_raw']", "value_expression": "summary" }, { "line": 745, "value_line": 745, "key": "reference_manual_summary", "target_expression": "extension['reference_manual_summary']", "value_expression": "summary" } ] }, { "symbol": "OrgbaseCrawler._crawl_all_countries", "path": "aggregator/orgbase_crawler.py", "line_start": 364, "line_end": 431, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "crawl", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 412, "receiver": "config", "key": "rate_limit_delay", "expression": "config.get('rate_limit_delay', 0.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._create_literature_object", "path": "aggregator/orgbase_crawler.py", "line_start": 2776, "line_end": 2793, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 2785, "value_line": 2785, "key": "raw_str", "target_expression": "new_object['raw_str']", "value_expression": "title_str" }, { "line": 2790, "value_line": 2790, "key": "title", "target_expression": "new_object['title']", "value_expression": "title_str" }, { "line": 2791, "value_line": 2791, "key": "type", "target_expression": "new_object['type']", "value_expression": "'unclassified'" } ] }, { "symbol": "OrgbaseCrawler._create_specifications_object", "path": "aggregator/orgbase_crawler.py", "line_start": 2810, "line_end": 2832, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 2831, "key": "dates", "value_expression": "[]" }, { "line": 2831, "key": "notes", "value_expression": "[]" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_activities", "path": "aggregator/orgbase_crawler.py", "line_start": 2536, "line_end": 2577, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 2538, "expression": "table.find_all('td')" }, { "line": 2539, "expression": "table.find_all('tr')" }, { "line": 2540, "expression": "trs[0].find_all('td')" } ], "dictionary_entries": [ { "line": 2543, "key": "events", "value_expression": "[]" }, { "line": 2561, "key": "id", "value_expression": "f'event-{_id}'" }, { "line": 2562, "key": "dates", "value_expression": "[year, '']" }, { "line": 2563, "key": "type", "value_expression": "'activity'" }, { "line": 2564, "key": "involved", "value_expression": "[{'name': involved_name, 'activities': [activity], 'dates': [year, '']}]" }, { "line": 2565, "key": "activities", "value_expression": "[activity]" }, { "line": 2566, "key": "identifiers", "value_expression": "[{'type': identifier_type if identifier_type and identifier else '', 'id': identifier if identifier else ''}]" }, { "line": 2572, "key": "notes", "value_expression": "['']" }, { "line": 2564, "key": "name", "value_expression": "involved_name" }, { "line": 2564, "key": "activities", "value_expression": "[activity]" }, { "line": 2564, "key": "dates", "value_expression": "[year, '']" }, { "line": 2568, "key": "type", "value_expression": "identifier_type if identifier_type and identifier else ''" }, { "line": 2569, "key": "id", "value_expression": "identifier if identifier else ''" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_address", "path": "aggregator/orgbase_crawler.py", "line_start": 1345, "line_end": 1396, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_address_line_after_label", "_coordinates_from_map_url", "_google_maps_q", "_parse_address_components" ], "literal_get_calls": [ { "line": 1375, "receiver": "maps_link", "key": "href", "expression": "maps_link.get('href')" }, { "line": 1389, "receiver": "website_link", "key": "href", "expression": "website_link.get('href', '')" } ], "html_queries": [ { "line": 1354, "expression": "soup.find(string=re.compile('Municipal:'))" }, { "line": 1358, "expression": "soup.find(string=re.compile('Region:'))" }, { "line": 1362, "expression": "soup.find(string=re.compile('Address:'))" }, { "line": 1385, "expression": "soup.find(string=re.compile('Website:'))" } ], "dictionary_entries": [ { "line": 1394, "key": "locations", "value_expression": "[locations_object]" }, { "line": 1378, "key": "google_maps_url", "value_expression": "map_url" }, { "line": 1381, "key": "type", "value_expression": "'google_maps_q'" }, { "line": 1381, "key": "value", "value_expression": "google_maps_q" } ], "subscript_assignments": [ { "line": 1356, "value_line": 1356, "key": "municipality", "target_expression": "locations_object['municipality']", "value_expression": "municipal_tag.split(':')[-1].strip()" }, { "line": 1360, "value_line": 1360, "key": "region", "target_expression": "locations_object['region']", "value_expression": "region_tag.split(':')[-1].strip()" }, { "line": 1393, "value_line": 1393, "key": "status", "target_expression": "locations_object['status']", "value_expression": "'latest_location'" }, { "line": 1373, "value_line": 1373, "key": "raw_str", "target_expression": "locations_object['raw_str']", "value_expression": "raw_address" }, { "line": 1390, "value_line": 1390, "key": "website", "target_expression": "locations_object['website']", "value_expression": "website_link['href']" }, { "line": 1380, "value_line": 1380, "key": "google_maps_q", "target_expression": "orgbase_extension['google_maps_q']", "value_expression": "google_maps_q" } ] }, { "symbol": "OrgbaseCrawler._extract_builder_info", "path": "aggregator/orgbase_crawler.py", "line_start": 1298, "line_end": 1343, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 1303, "expression": "soup.find('i', string=re.compile('Built by:'))" } ], "dictionary_entries": [ { "line": 1302, "key": "involved", "value_expression": "[]" }, { "line": 1325, "key": "name", "value_expression": "builder.strip()" }, { "line": 1326, "key": "activities", "value_expression": "['builder']" }, { "line": 1327, "key": "dates", "value_expression": "[year]" }, { "line": 1328, "key": "opus_number", "value_expression": "int(opus)" }, { "line": 1329, "key": "raw_str", "value_expression": "full_text.strip()" }, { "line": 1335, "key": "name", "value_expression": "builder.strip()" }, { "line": 1336, "key": "activities", "value_expression": "['builder']" }, { "line": 1337, "key": "dates", "value_expression": "dates" }, { "line": 1338, "key": "notes", "value_expression": "notes" }, { "line": 1339, "key": "raw_str", "value_expression": "full_text.strip()" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_compound_stops", "path": "aggregator/orgbase_crawler.py", "line_start": 2579, "line_end": 2619, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 2582, "expression": "table.find_all('tr')" }, { "line": 2586, "expression": "row.find_all('td')" } ], "dictionary_entries": [ { "line": 2619, "key": "specifications", "value_expression": "{'compound_stops': compound_stops}" }, { "line": 2619, "key": "compound_stops", "value_expression": "compound_stops" }, { "line": 2608, "key": "id", "value_expression": "f'compound-{ct_row}'" }, { "line": 2609, "key": "name", "value_expression": "name" }, { "line": 2610, "key": "compositions", "value_expression": "compositions" }, { "line": 2611, "key": "raw_str", "value_expression": "raw_composition" }, { "line": 2612, "key": "dates", "value_expression": "dates" }, { "line": 2613, "key": "notes", "value_expression": "notes" }, { "line": 2603, "key": "note", "value_expression": "note.strip() if type(note) == str else note" }, { "line": 2603, "key": "ranks", "value_expression": "ranks" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_current_location", "path": "aggregator/orgbase_crawler.py", "line_start": 1205, "line_end": 1246, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_apply_country_to_location" ], "literal_get_calls": [], "html_queries": [ { "line": 1210, "expression": "soup.find('b', string=None)" }, { "line": 1213, "expression": "soup.find('b')" }, { "line": 1213, "expression": "soup.find('b').find('font', size='+1')" }, { "line": 1213, "expression": "soup.find('b')" } ], "dictionary_entries": [ { "line": 1239, "key": "status", "value_expression": "'current_location'" }, { "line": 1240, "key": "town", "value_expression": "town" }, { "line": 1241, "key": "building", "value_expression": "building" }, { "line": 1242, "key": "raw_str", "value_expression": "text" }, { "line": 1246, "key": "locations", "value_expression": "[location]" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_descriptions", "path": "aggregator/orgbase_crawler.py", "line_start": 1493, "line_end": 1541, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_orgbase_activity_table" ], "literal_get_calls": [], "html_queries": [ { "line": 1529, "expression": "sibling.find_all('li', recursive=False)" } ], "dictionary_entries": [ { "line": 1541, "key": "descriptions", "value_expression": "descriptions" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_different_specs", "path": "aggregator/orgbase_crawler.py", "line_start": 2453, "line_end": 2529, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_is_valid_werk_with_range" ], "literal_get_calls": [], "html_queries": [ { "line": 2454, "expression": "table.find_all('tr')" }, { "line": 2455, "expression": "rows[0].find_all('li')" }, { "line": 2499, "expression": "rows[0].find_all('td')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_identifiers", "path": "aggregator/orgbase_crawler.py", "line_start": 1446, "line_end": 1491, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 1464, "expression": "soup.find_all('font', size='-2')" } ], "dictionary_entries": [ { "line": 1453, "key": "Description nr.", "value_expression": "{'name': 'orgbase_id', 'host': 'Orgbase', 'hostcode': 'orgbase'}" }, { "line": 1454, "key": "Stichting Orgelcentrum card number", "value_expression": "{'name': 'Stichting Orgelcentrum card number', 'host': 'Stichting Orgelcentrum', 'hostcode': 'stichting_orgelcentrum'}" }, { "line": 1461, "key": "other_identifiers", "value_expression": "[]" }, { "line": 1453, "key": "name", "value_expression": "'orgbase_id'" }, { "line": 1453, "key": "host", "value_expression": "'Orgbase'" }, { "line": 1453, "key": "hostcode", "value_expression": "'orgbase'" }, { "line": 1455, "key": "name", "value_expression": "'Stichting Orgelcentrum card number'" }, { "line": 1456, "key": "host", "value_expression": "'Stichting Orgelcentrum'" }, { "line": 1457, "key": "hostcode", "value_expression": "'stichting_orgelcentrum'" }, { "line": 1478, "key": "name", "value_expression": "mapping['name']" }, { "line": 1479, "key": "id", "value_expression": "value.rstrip('.')" }, { "line": 1480, "key": "host", "value_expression": "mapping['host']" }, { "line": 1481, "key": "hostcode", "value_expression": "mapping['hostcode']" }, { "line": 1482, "key": "raw_str", "value_expression": "text" }, { "line": 1483, "key": "dates", "value_expression": "dates" }, { "line": 1484, "key": "notes", "value_expression": "notes" } ], "subscript_assignments": [ { "line": 1489, "value_line": 1489, "key": "orgbase_id", "target_expression": "outdata['orgbase_id']", "value_expression": "value.rstrip('.')" } ] }, { "symbol": "OrgbaseCrawler._extract_images_data", "path": "aggregator/orgbase_crawler.py", "line_start": 2621, "line_end": 2692, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [ { "line": 2667, "receiver": "original_a_tag", "key": "title", "expression": "original_a_tag.get('title', '')" }, { "line": 2671, "receiver": "img_tag", "key": "title", "expression": "img_tag.get('title', '')" }, { "line": 2671, "receiver": "img_tag", "key": "alt", "expression": "img_tag.get('alt', '')" } ], "html_queries": [ { "line": 2633, "expression": "soup.find_all('a', href=re.compile('\\\\.(jpg|jpeg|png|tiff)$', re.IGNORECASE))" }, { "line": 2662, "expression": "soup.find('a', href=lambda h: h and urljoin(BASE_URL, h) == link)" }, { "line": 2669, "expression": "original_a_tag.find('img')" } ], "dictionary_entries": [ { "line": 2692, "key": "media", "value_expression": "{'images': self._tmp_data['images']}" }, { "line": 2692, "key": "images", "value_expression": "self._tmp_data['images']" } ], "subscript_assignments": [ { "line": 2640, "value_line": 2640, "key": "images", "target_expression": "self._tmp_data['images']", "value_expression": "[]" }, { "line": 2655, "value_line": 2655, "key": "filename", "target_expression": "new_object['filename']", "value_expression": "Path(link).name" }, { "line": 2656, "value_line": 2656, "key": "host", "target_expression": "new_object['host']", "value_expression": "parsed_link.netloc" }, { "line": 2657, "value_line": 2657, "key": "link", "target_expression": "new_object['link']", "value_expression": "link" }, { "line": 2658, "value_line": 2658, "key": "available", "target_expression": "new_object['available']", "value_expression": "True" }, { "line": 2684, "value_line": 2684, "key": "source", "target_expression": "new_object['source']", "value_expression": "clean_text" }, { "line": 2685, "value_line": 2685, "key": "copyright", "target_expression": "new_object['copyright']", "value_expression": "clean_text" } ] }, { "symbol": "OrgbaseCrawler._extract_literature_section", "path": "aggregator/orgbase_crawler.py", "line_start": 2694, "line_end": 2774, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_create_literature_object" ], "literal_get_calls": [ { "line": 2734, "receiver": "a_tag", "key": "href", "expression": "a_tag.get('href')" } ], "html_queries": [ { "line": 2701, "expression": "table.find_all('tr')" }, { "line": 2709, "expression": "row.find_all('td')" }, { "line": 2720, "expression": "content_cell.find_all('li')" }, { "line": 2732, "expression": "content_cell.find_all('a', href=True)" }, { "line": 2726, "expression": "content_cell.find_all('li')" } ], "dictionary_entries": [ { "line": 2762, "key": "type", "value_expression": "''" }, { "line": 2763, "key": "host", "value_expression": "parsed_link.netloc" }, { "line": 2764, "key": "link", "value_expression": "link" }, { "line": 2765, "key": "linktext", "value_expression": "linktext" }, { "line": 2766, "key": "available", "value_expression": "True" }, { "line": 2767, "key": "archive_link", "value_expression": "''" } ], "subscript_assignments": [ { "line": 2741, "value_line": 2741, "key": "literature", "target_expression": "data['literature']", "value_expression": "[self._create_literature_object(title) for title in dict.fromkeys(all_literature_titles)]" }, { "line": 2748, "value_line": 2748, "key": "recordings", "target_expression": "data['recordings']", "value_expression": "[self._create_literature_object(title) for title in dict.fromkeys(all_recordings_titles)]" }, { "line": 2755, "value_line": 2755, "key": "links", "target_expression": "data['links']", "value_expression": "[]" } ] }, { "symbol": "OrgbaseCrawler._extract_location_history", "path": "aggregator/orgbase_crawler.py", "line_start": 1398, "line_end": 1444, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 1408, "expression": "soup.find(string='Short list of all locations the organ has been')" }, { "line": 1417, "expression": "table.find_all('tr')" }, { "line": 1418, "expression": "row.find_all(['td', 'th'])" } ], "dictionary_entries": [ { "line": 1416, "key": "locations", "value_expression": "[]" }, { "line": 1431, "key": "status", "value_expression": "'previous_location'" }, { "line": 1432, "key": "dates", "value_expression": "dates" }, { "line": 1433, "key": "municipality", "value_expression": "place" }, { "line": 1434, "key": "building", "value_expression": "building_name" }, { "line": 1435, "key": "notes", "value_expression": "building_notes" }, { "line": 1436, "key": "extensions", "value_expression": "{'orgbase': {'location_history_row_index': source_row_index}}" }, { "line": 1437, "key": "orgbase", "value_expression": "{'location_history_row_index': source_row_index}" }, { "line": 1438, "key": "location_history_row_index", "value_expression": "source_row_index" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_recordings", "path": "aggregator/orgbase_crawler.py", "line_start": 2795, "line_end": 2796, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_technical_data", "path": "aggregator/orgbase_crawler.py", "line_start": 2798, "line_end": 2808, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_structure_technical_data" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._extract_title_image", "path": "aggregator/orgbase_crawler.py", "line_start": 1248, "line_end": 1296, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [ { "line": 1277, "receiver": "title_image_tag.img", "key": "alt", "expression": "title_image_tag.img.get('alt', '')" }, { "line": 1264, "receiver": "title_image_tag", "key": "href", "expression": "title_image_tag.get('href', '')" }, { "line": 1268, "receiver": "img_tag", "key": "src", "expression": "img_tag.get('src', '')" } ], "html_queries": [ { "line": 1256, "expression": "soup.select_one('h1 + br + a')" }, { "line": 1267, "expression": "title_image_tag.find('img', src=True)" }, { "line": 1257, "expression": "title_image_tag.find('img')" } ], "dictionary_entries": [], "subscript_assignments": [ { "line": 1274, "value_line": 1274, "key": "media", "target_expression": "outdata['media']", "value_expression": "clean_template(self.orgbase_template['media'])" }, { "line": 1279, "value_line": 1279, "key": "filename", "target_expression": "image_object['filename']", "value_expression": "Path(link).name" }, { "line": 1280, "value_line": 1280, "key": "host", "target_expression": "image_object['host']", "value_expression": "str(urlparse(link).netloc)" }, { "line": 1281, "value_line": 1281, "key": "link", "target_expression": "image_object['link']", "value_expression": "link" }, { "line": 1295, "value_line": 1295, "key": "images", "target_expression": "self._tmp_data['images']", "value_expression": "outdata['media']['images']" }, { "line": 1291, "value_line": 1291, "key": "source", "target_expression": "image_object['source']", "value_expression": "linktext" }, { "line": 1292, "value_line": 1292, "key": "copyright", "target_expression": "image_object['copyright']", "value_expression": "linktext" } ] }, { "symbol": "OrgbaseCrawler._get_literature", "path": "aggregator/orgbase_crawler.py", "line_start": 2933, "line_end": 2949, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._get_recordings", "path": "aggregator/orgbase_crawler.py", "line_start": 2980, "line_end": 2990, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 2983, "expression": "soup.find('td', valign='TOP', string='Recordings')" }, { "line": 2985, "expression": "recording_section.find_next_sibling('td').find('ul')" }, { "line": 2987, "expression": "ul_element.find_all('li')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._get_stops", "path": "aggregator/orgbase_crawler.py", "line_start": 2873, "line_end": 2883, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._get_technical_data", "path": "aggregator/orgbase_crawler.py", "line_start": 2834, "line_end": 2846, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._is_valid_werk_with_range", "path": "aggregator/orgbase_crawler.py", "line_start": 2531, "line_end": 2534, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._merge_media_sections", "path": "aggregator/orgbase_crawler.py", "line_start": 791, "line_end": 810, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_merge_unique_list" ], "literal_get_calls": [ { "line": 799, "receiver": "media_data", "key": "videos", "expression": "media_data.get('videos')" }, { "line": 803, "receiver": "media_data", "key": "audio", "expression": "media_data.get('audio')" }, { "line": 807, "receiver": "media_data", "key": "documents", "expression": "media_data.get('documents')" }, { "line": 801, "receiver": "complete_data['media']", "key": "videos", "expression": "complete_data['media'].get('videos', [])" }, { "line": 801, "receiver": "media_data", "key": "videos", "expression": "media_data.get('videos', [])" }, { "line": 805, "receiver": "complete_data['media']", "key": "audio", "expression": "complete_data['media'].get('audio', [])" }, { "line": 805, "receiver": "media_data", "key": "audio", "expression": "media_data.get('audio', [])" }, { "line": 809, "receiver": "complete_data['media']", "key": "documents", "expression": "complete_data['media'].get('documents', [])" }, { "line": 809, "receiver": "media_data", "key": "documents", "expression": "media_data.get('documents', [])" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 797, "value_line": 797, "key": "media", "target_expression": "complete_data['media']", "value_expression": "{}" }, { "line": 800, "value_line": 800, "key": "videos", "target_expression": "complete_data['media']['videos']", "value_expression": "self._merge_unique_list(complete_data['media'].get('videos', []), media_data.get('videos', []))" }, { "line": 804, "value_line": 804, "key": "audio", "target_expression": "complete_data['media']['audio']", "value_expression": "self._merge_unique_list(complete_data['media'].get('audio', []), media_data.get('audio', []))" }, { "line": 808, "value_line": 808, "key": "documents", "target_expression": "complete_data['media']['documents']", "value_expression": "self._merge_unique_list(complete_data['media'].get('documents', []), media_data.get('documents', []))" } ] }, { "symbol": "OrgbaseCrawler._normalize_country", "path": "aggregator/orgbase_crawler.py", "line_start": 1074, "line_end": 1090, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._parse_address_components", "path": "aggregator/orgbase_crawler.py", "line_start": 1099, "line_end": 1149, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_looks_like_house_number", "_looks_like_postcode" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 1118, "value_line": 1118, "key": "address", "target_expression": "parsed['address']", "value_expression": "', '.join(parts[:separated_postcode_index])" }, { "line": 1119, "value_line": 1119, "key": "postcode", "target_expression": "parsed['postcode']", "value_expression": "parts[separated_postcode_index]" }, { "line": 1120, "value_line": 1120, "key": "town", "target_expression": "parsed['town']", "value_expression": "', '.join(parts[separated_postcode_index + 1:])" }, { "line": 1137, "value_line": 1137, "key": "address", "target_expression": "parsed['address']", "value_expression": "match.group('address').strip(' ,')" }, { "line": 1138, "value_line": 1138, "key": "postcode", "target_expression": "parsed['postcode']", "value_expression": "match.group('postcode').strip()" }, { "line": 1139, "value_line": 1139, "key": "town", "target_expression": "parsed['town']", "value_expression": "match.group('town').strip()" }, { "line": 1143, "value_line": 1143, "key": "address", "target_expression": "parsed['address']", "value_expression": "parts[0]" }, { "line": 1147, "value_line": 1147, "key": "address", "target_expression": "parsed['address']", "value_expression": "address_text" }, { "line": 1126, "value_line": 1126, "key": "address", "target_expression": "parsed['address']", "value_expression": "', '.join(parts[:index])" }, { "line": 1127, "value_line": 1127, "key": "postcode", "target_expression": "parsed['postcode']", "value_expression": "match.group('postcode').strip()" }, { "line": 1128, "value_line": 1128, "key": "town", "target_expression": "parsed['town']", "value_expression": "match.group('town').strip()" }, { "line": 1145, "value_line": 1145, "key": "town", "target_expression": "parsed['town']", "value_expression": "parts[1]" } ] }, { "symbol": "OrgbaseCrawler._process_accessories_string", "path": "aggregator/orgbase_crawler.py", "line_start": 2174, "line_end": 2212, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 2208, "value_line": 2208, "key": "name", "target_expression": "new_acc_data['name']", "value_expression": "name" }, { "line": 2209, "value_line": 2209, "key": "dates", "target_expression": "new_acc_data['dates']", "value_expression": "dates" }, { "line": 2210, "value_line": 2210, "key": "raw_str", "target_expression": "new_acc_data['raw_str']", "value_expression": "input_str" }, { "line": 2188, "value_line": 2188, "key": "notes", "target_expression": "new_acc_data['notes']", "value_expression": "[parentheses_detected]" } ] }, { "symbol": "OrgbaseCrawler._process_couplers", "path": "aggregator/orgbase_crawler.py", "line_start": 2091, "line_end": 2172, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_split_coupler_string_by_nlp" ], "literal_get_calls": [ { "line": 2167, "receiver": "coupler_data", "key": "type", "expression": "coupler_data.get('type')" }, { "line": 2132, "receiver": "handler", "key": "destination", "expression": "handler.get('destination')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2101, "key": "tirasse", "value_expression": "{'source': 'manual', 'destination': 'Pédale', 'type': 'unison'}" }, { "line": 2103, "key": "pedalkoppel", "value_expression": "{'source': 'manual', 'destination': 'Pédale', 'type': 'unison'}" }, { "line": 2105, "key": "suboktavkoppel", "value_expression": "{'type': 'sub_octave'}" }, { "line": 2106, "key": "sub-oktav", "value_expression": "{'type': 'sub_octave'}" }, { "line": 2107, "key": "sub", "value_expression": "{'type': 'sub_octave'}" }, { "line": 2108, "key": "superoktavkoppel", "value_expression": "{'type': 'super_octave'}" }, { "line": 2109, "key": "super-oktav", "value_expression": "{'type': 'super_octave'}" }, { "line": 2110, "key": "super", "value_expression": "{'type': 'super_octave'}" }, { "line": 2111, "key": "acouplement", "value_expression": "{'type': 'unison'}" }, { "line": 2112, "key": "copula", "value_expression": "{'type': 'unison'}" }, { "line": 2101, "key": "source", "value_expression": "'manual'" }, { "line": 2101, "key": "destination", "value_expression": "'Pédale'" }, { "line": 2101, "key": "type", "value_expression": "'unison'" }, { "line": 2103, "key": "source", "value_expression": "'manual'" }, { "line": 2103, "key": "destination", "value_expression": "'Pédale'" }, { "line": 2103, "key": "type", "value_expression": "'unison'" }, { "line": 2105, "key": "type", "value_expression": "'sub_octave'" }, { "line": 2106, "key": "type", "value_expression": "'sub_octave'" }, { "line": 2107, "key": "type", "value_expression": "'sub_octave'" }, { "line": 2108, "key": "type", "value_expression": "'super_octave'" }, { "line": 2109, "key": "type", "value_expression": "'super_octave'" }, { "line": 2110, "key": "type", "value_expression": "'super_octave'" }, { "line": 2111, "key": "type", "value_expression": "'unison'" }, { "line": 2112, "key": "type", "value_expression": "'unison'" } ], "subscript_assignments": [ { "line": 2117, "value_line": 2117, "key": "id", "target_expression": "coupler_data['id']", "value_expression": "f'coupler-{i + 1}'" }, { "line": 2118, "value_line": 2118, "key": "raw_str", "target_expression": "coupler_data['raw_str']", "value_expression": "pair_str" }, { "line": 2157, "value_line": 2157, "key": "source", "target_expression": "coupler_data['source']", "value_expression": "splitted[0]" }, { "line": 2158, "value_line": 2158, "key": "destination", "target_expression": "coupler_data['destination']", "value_expression": "splitted[1].rstrip('.').strip()" }, { "line": 2168, "value_line": 2168, "key": "type", "target_expression": "coupler_data['type']", "value_expression": "'undefined'" }, { "line": 2130, "value_line": 2130, "key": "type", "target_expression": "coupler_data['type']", "value_expression": "handler['type']" }, { "line": 2163, "value_line": 2163, "key": "source", "target_expression": "coupler_data['source']", "value_expression": "splitted[0].rstrip('.').strip()" }, { "line": 2164, "value_line": 2164, "key": "destination", "target_expression": "coupler_data['destination']", "value_expression": "'unknown'" }, { "line": 2133, "value_line": 2133, "key": "source", "target_expression": "coupler_data['source']", "value_expression": "remaining_str" }, { "line": 2134, "value_line": 2134, "key": "destination", "target_expression": "coupler_data['destination']", "value_expression": "'Pédale'" }, { "line": 2139, "value_line": 2139, "key": "source", "target_expression": "coupler_data['source']", "value_expression": "source" }, { "line": 2140, "value_line": 2140, "key": "destination", "target_expression": "coupler_data['destination']", "value_expression": "destination" } ] }, { "symbol": "OrgbaseCrawler._process_stop_specs", "path": "aggregator/orgbase_crawler.py", "line_start": 1901, "line_end": 2029, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [ { "line": 1996, "receiver": "stop_spec_data", "key": "range_mapped", "expression": "stop_spec_data.get('range_mapped')" }, { "line": 2021, "receiver": "self._tmp_data", "key": "stop_action", "expression": "self._tmp_data.get('stop_action', '')" }, { "line": 1912, "receiver": "self._tmp_data", "key": "stop_action", "expression": "self._tmp_data.get('stop_action', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 2017, "value_line": 2017, "key": "name", "target_expression": "stop_spec_data['name']", "value_expression": "name" }, { "line": 2018, "value_line": 2018, "key": "werk", "target_expression": "stop_spec_data['werk']", "value_expression": "werk" }, { "line": 2019, "value_line": 2019, "key": "dates", "target_expression": "stop_spec_data['dates']", "value_expression": "dates" }, { "line": 2020, "value_line": 2020, "key": "notes", "target_expression": "stop_spec_data['notes']", "value_expression": "notes" }, { "line": 2021, "value_line": 2021, "key": "action", "target_expression": "stop_spec_data['action']", "value_expression": "self._tmp_data.get('stop_action', '')" }, { "line": 2027, "value_line": 2027, "key": "cleaned", "target_expression": "stop_spec_data['cleaned']", "value_expression": "cleaned_name" }, { "line": 1910, "value_line": 1910, "key": "name", "target_expression": "stop_spec_data['name']", "value_expression": "'Not Available / Unknown'" }, { "line": 1911, "value_line": 1911, "key": "werk", "target_expression": "stop_spec_data['werk']", "value_expression": "werk" }, { "line": 1912, "value_line": 1912, "key": "action", "target_expression": "stop_spec_data['action']", "value_expression": "self._tmp_data.get('stop_action', '')" }, { "line": 1913, "value_line": 1913, "key": "notes", "target_expression": "stop_spec_data['notes']", "value_expression": "['N/A']" }, { "line": 1928, "value_line": 1928, "key": "fractal", "target_expression": "stop_spec_data['fractal']", "value_expression": "fractal_detected" }, { "line": 1938, "value_line": 1938, "key": "mixture", "target_expression": "stop_spec_data['mixture']", "value_expression": "clean_string_by_ls_elements(mixture_detected, mixture_identifiers)" }, { "line": 1983, "value_line": 1983, "key": "length", "target_expression": "stop_spec_data['length']", "value_expression": "primary_length" }, { "line": 1984, "value_line": 1984, "key": "fractal", "target_expression": "stop_spec_data['fractal']", "value_expression": "primary_fractal" }, { "line": 1997, "value_line": 1997, "key": "transposition_sounding", "target_expression": "stop_spec_data['transposition_sounding']", "value_expression": "int(stop_spec_data['range_mapped'][0] - 36)" }, { "line": 2002, "value_line": 2002, "key": "region", "target_expression": "split_object['region']", "value_expression": "'bass'" }, { "line": 2003, "value_line": 2003, "key": "range", "target_expression": "split_object['range']", "value_expression": "[36, 59]" }, { "line": 2006, "value_line": 2006, "key": "region", "target_expression": "split_object['region']", "value_expression": "'discant'" }, { "line": 2007, "value_line": 2007, "key": "range", "target_expression": "split_object['range']", "value_expression": "[60, 96]" }, { "line": 1986, "value_line": 1986, "key": "range_mapped", "target_expression": "stop_spec_data['range_mapped']", "value_expression": "convert_pipelength_to_midi(length=primary_length, fractal=primary_fractal)" } ] }, { "symbol": "OrgbaseCrawler._process_stops", "path": "aggregator/orgbase_crawler.py", "line_start": 1863, "line_end": 1899, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_process_stop_specs" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 1889, "value_line": 1890, "key": "id", "target_expression": "stops_data['id']", "value_expression": "f'stop-{len(all_stops) + 1}' if not different_spec else f'df-{ct_diff}-stop-{len(all_stops) + 1}'" }, { "line": 1892, "value_line": 1892, "key": "raw_str", "target_expression": "stops_data['raw_str']", "value_expression": "stop" } ] }, { "symbol": "OrgbaseCrawler._process_werkname", "path": "aggregator/orgbase_crawler.py", "line_start": 2313, "line_end": 2451, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_get_manual_position", "_get_short_octave_object" ], "literal_get_calls": [ { "line": 2415, "receiver": "self._tmp_data", "key": "key_action", "expression": "self._tmp_data.get('key_action', '')" }, { "line": 2436, "receiver": "self._tmp_data", "key": "key_action", "expression": "self._tmp_data.get('key_action', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2370, "key": "between", "value_expression": "between" } ], "subscript_assignments": [ { "line": 2337, "value_line": 2337, "key": "id", "target_expression": "new_object['id']", "value_expression": "f'werk-{ct_werk}' if not different_spec else f'df-{ct_diff}-werk-{ct_werk}'" }, { "line": 2339, "value_line": 2339, "key": "name", "target_expression": "new_object['name']", "value_expression": "cleaned_werkname" }, { "line": 2402, "value_line": 2402, "key": "dates", "target_expression": "new_object['dates']", "value_expression": "dates" }, { "line": 2403, "value_line": 2403, "key": "notes", "target_expression": "new_object['notes']", "value_expression": "notes" }, { "line": 2327, "value_line": 2327, "key": "werke", "target_expression": "self._tmp_data['werke']", "value_expression": "[]" }, { "line": 2333, "value_line": 2333, "key": "werke", "target_expression": "self._tmp_data['different_specs'][ct_diff]['werke']", "value_expression": "[]" }, { "line": 2396, "value_line": 2396, "key": "range", "target_expression": "new_object['range']", "value_expression": "note_range_midi" }, { "line": 2411, "value_line": 2412, "key": "id", "target_expression": "new_manual_object['id']", "value_expression": "f'manual-{ct_werk}' if not different_spec else f'df-{ct_diff}-manual-{ct_werk}'" }, { "line": 2414, "value_line": 2414, "key": "name", "target_expression": "new_manual_object['name']", "value_expression": "cleaned_werkname" }, { "line": 2415, "value_line": 2415, "key": "action", "target_expression": "new_manual_object['action']", "value_expression": "self._tmp_data.get('key_action', '') if not different_spec else ''" }, { "line": 2416, "value_line": 2416, "key": "range", "target_expression": "new_manual_object['range']", "value_expression": "note_range_midi if note_range_midi else [36, 96]" }, { "line": 2417, "value_line": 2417, "key": "dates", "target_expression": "new_manual_object['dates']", "value_expression": "dates" }, { "line": 2418, "value_line": 2418, "key": "notes", "target_expression": "new_manual_object['notes']", "value_expression": "notes" }, { "line": 2420, "value_line": 2420, "key": "position", "target_expression": "new_manual_object['position']", "value_expression": "position if position else str(ct_werk)" }, { "line": 2434, "value_line": 2434, "key": "id", "target_expression": "new_pedal_object['id']", "value_expression": "f'pedal-{ct_werk}' if not different_spec else f'df-{ct_diff}-pedal-{ct_werk}'" }, { "line": 2435, "value_line": 2435, "key": "name", "target_expression": "new_pedal_object['name']", "value_expression": "cleaned_werkname" }, { "line": 2436, "value_line": 2436, "key": "action", "target_expression": "new_pedal_object['action']", "value_expression": "self._tmp_data.get('key_action', '') if not different_spec else ''" }, { "line": 2437, "value_line": 2437, "key": "range", "target_expression": "new_pedal_object['range']", "value_expression": "note_range_midi if note_range_midi else [36, 67]" }, { "line": 2438, "value_line": 2438, "key": "dates", "target_expression": "new_pedal_object['dates']", "value_expression": "dates" }, { "line": 2439, "value_line": 2439, "key": "notes", "target_expression": "new_pedal_object['notes']", "value_expression": "notes" }, { "line": 2441, "value_line": 2441, "key": "position", "target_expression": "new_pedal_object['position']", "value_expression": "position if position else str(ct_werk)" }, { "line": 2386, "value_line": 2386, "key": "short_octave", "target_expression": "new_object['short_octave']", "value_expression": "short_octave_object" }, { "line": 2423, "value_line": 2423, "key": "manuals", "target_expression": "self._tmp_data['manuals']", "value_expression": "[]" }, { "line": 2427, "value_line": 2427, "key": "manuals", "target_expression": "self._tmp_data['different_specs'][ct_diff]['manuals']", "value_expression": "[]" }, { "line": 2444, "value_line": 2444, "key": "pedals", "target_expression": "self._tmp_data['pedals']", "value_expression": "[]" }, { "line": 2448, "value_line": 2448, "key": "pedals", "target_expression": "self._tmp_data['different_specs'][ct_diff]['pedals']", "value_expression": "[]" } ] }, { "symbol": "OrgbaseCrawler._split_coupler_string_by_nlp", "path": "aggregator/orgbase_crawler.py", "line_start": 2043, "line_end": 2089, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._split_specification_values", "path": "aggregator/orgbase_crawler.py", "line_start": 1761, "line_end": 1791, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler._structure_technical_data", "path": "aggregator/orgbase_crawler.py", "line_start": 944, "line_end": 1047, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 977, "expression": "table.find_all('tr')" }, { "line": 953, "expression": "soup.find('b', string='Technical data')" }, { "line": 979, "expression": "row.find_all('td')" } ], "dictionary_entries": [ { "line": 960, "key": "specifications", "value_expression": "{}" }, { "line": 964, "key": "Total number of stops", "value_expression": "'stop_count'" }, { "line": 965, "key": "Key action", "value_expression": "'key_action'" }, { "line": 966, "key": "Stop action", "value_expression": "'stop_action'" }, { "line": 967, "key": "Windchest(s)", "value_expression": "'windchests'" }, { "line": 968, "key": "Temperament", "value_expression": "'temperament'" }, { "line": 969, "key": "Pitch", "value_expression": "'pitch'" }, { "line": 970, "key": "Wind pressure", "value_expression": "'wind_pressure'" }, { "line": 971, "key": "Number of stops per division", "value_expression": "'stops_per_division'" }, { "line": 999, "key": "division", "value_expression": "division_name" }, { "line": 1000, "key": "count", "value_expression": "int(division_count_match.group(0)) if division_count_match else value" }, { "line": 1001, "key": "raw_str", "value_expression": "raw_row" }, { "line": 1005, "key": "label", "value_expression": "label" }, { "line": 1005, "key": "value", "value_expression": "value" }, { "line": 1005, "key": "raw_str", "value_expression": "raw_row" }, { "line": 1018, "key": "division", "value_expression": "''" }, { "line": 1018, "key": "count", "value_expression": "value" }, { "line": 1018, "key": "raw_str", "value_expression": "raw_row" }, { "line": 1037, "key": "label", "value_expression": "label" }, { "line": 1037, "key": "value", "value_expression": "value" }, { "line": 1037, "key": "raw_str", "value_expression": "raw_row" } ], "subscript_assignments": [ { "line": 1040, "value_line": 1040, "key": "stops_per_division", "target_expression": "specs['stops_per_division']", "value_expression": "stops_per_division" }, { "line": 1042, "value_line": 1042, "key": "technical_rows_unstructured", "target_expression": "specs['technical_rows_unstructured']", "value_expression": "unstructured_rows" }, { "line": 1023, "value_line": 1023, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "int(match.group(0))" }, { "line": 1025, "value_line": 1025, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "value" } ] }, { "symbol": "OrgbaseCrawler.crawl", "path": "aggregator/orgbase_crawler.py", "line_start": 209, "line_end": 362, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_crawl_all_countries", "_is_explicit_empty_result", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 212, "receiver": "kwargs", "key": "country", "expression": "kwargs.get('country')" }, { "line": 213, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" } ], "html_queries": [ { "line": 261, "expression": "soup.find_all('tr')" }, { "line": 279, "expression": "row.find_all('td')" }, { "line": 282, "expression": "cols[0].find('a')" } ], "dictionary_entries": [ { "line": 299, "key": "orgbase_id", "value_expression": "orgbase_id" }, { "line": 300, "key": "url", "value_expression": "urljoin(self.base_url, link_tag['href'])" }, { "line": 301, "key": "country", "value_expression": "country" }, { "line": 302, "key": "town", "value_expression": "town" }, { "line": 303, "key": "building", "value_expression": "building" }, { "line": 304, "key": "builder", "value_expression": "builder" }, { "line": 305, "key": "manuals", "value_expression": "manuals" }, { "line": 306, "key": "scraped_at", "value_expression": "datetime.now()" } ], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler.get_compound_stop", "path": "aggregator/orgbase_crawler.py", "line_start": 3004, "line_end": 3005, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler.get_other_specifications", "path": "aggregator/orgbase_crawler.py", "line_start": 3001, "line_end": 3002, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler.offline_transformer", "path": "aggregator/orgbase_crawler.py", "line_start": 105, "line_end": 114, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "__new__" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgbaseCrawler.transform", "path": "aggregator/orgbase_crawler.py", "line_start": 619, "line_end": 726, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "_apply_country_to_location", "_apply_reference_manual_summary", "_clean_empty_fields", "_extract_images_data", "_merge_media_sections", "_merge_unique_list", "_set_source_snapshot", "_structure_footer_data", "_structure_main_data", "_structure_other_data", "_structure_table_data", "_structure_technical_data" ], "literal_get_calls": [ { "line": 626, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html', '')" }, { "line": 644, "receiver": "footer_data", "key": "literature", "expression": "footer_data.get('literature')" }, { "line": 648, "receiver": "footer_data", "key": "other_sources", "expression": "footer_data.get('other_sources')" }, { "line": 655, "receiver": "raw_data", "key": "orgbase_id", "expression": "raw_data.get('orgbase_id')" }, { "line": 652, "receiver": "footer_data", "key": "media", "expression": "footer_data.get('media', {})" }, { "line": 660, "receiver": "complete_data", "key": "locations", "expression": "complete_data.get('locations')" }, { "line": 680, "receiver": "raw_data", "key": "imported_cache_id", "expression": "raw_data.get('imported_cache_id')" }, { "line": 680, "receiver": "raw_data", "key": "orgbase_id", "expression": "raw_data.get('orgbase_id')" }, { "line": 683, "receiver": "raw_data", "key": "imported_cache_id", "expression": "raw_data.get('imported_cache_id')" }, { "line": 710, "receiver": "raw_data", "key": "scraped_at", "expression": "raw_data.get('scraped_at')" }, { "line": 646, "receiver": "complete_data", "key": "literature", "expression": "complete_data.get('literature', [])" }, { "line": 646, "receiver": "footer_data", "key": "literature", "expression": "footer_data.get('literature', [])" }, { "line": 650, "receiver": "complete_data", "key": "other_sources", "expression": "complete_data.get('other_sources', [])" }, { "line": 650, "receiver": "footer_data", "key": "other_sources", "expression": "footer_data.get('other_sources', [])" }, { "line": 663, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 664, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 666, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 671, "receiver": "location", "key": "town", "expression": "location.get('town')" }, { "line": 672, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 673, "receiver": "location", "key": "building", "expression": "location.get('building')" }, { "line": 674, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 675, "receiver": "location", "key": "country", "expression": "location.get('country')" }, { "line": 714, "receiver": "raw_data", "key": "orgbase_id", "expression": "raw_data.get('orgbase_id')" }, { "line": 718, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 719, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" }, { "line": 720, "receiver": "raw_data", "key": "artifact_ids", "expression": "raw_data.get('artifact_ids')" }, { "line": 721, "receiver": "raw_data", "key": "normalized_at", "expression": "raw_data.get('normalized_at')" }, { "line": 723, "receiver": "normalized", "key": "locations", "expression": "normalized.get('locations')" }, { "line": 676, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 715, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 715, "receiver": "raw_data", "key": "link", "expression": "raw_data.get('link')" } ], "html_queries": [], "dictionary_entries": [ { "line": 662, "key": "status", "value_expression": "'current_location'" }, { "line": 663, "key": "town", "value_expression": "raw_data.get('town')" }, { "line": 664, "key": "building", "value_expression": "raw_data.get('building')" } ], "subscript_assignments": [ { "line": 655, "value_line": 655, "key": "orgbase_id", "target_expression": "complete_data['orgbase_id']", "value_expression": "raw_data.get('orgbase_id')" }, { "line": 710, "value_line": 710, "key": "retrieved_at", "target_expression": "cleaned_data.setdefault('source_info', {})['retrieved_at']", "value_expression": "raw_data.get('scraped_at') or ''" }, { "line": 645, "value_line": 645, "key": "literature", "target_expression": "complete_data['literature']", "value_expression": "self._merge_unique_list(complete_data.get('literature', []), footer_data.get('literature', []))" }, { "line": 649, "value_line": 649, "key": "other_sources", "target_expression": "complete_data['other_sources']", "value_expression": "self._merge_unique_list(complete_data.get('other_sources', []), footer_data.get('other_sources', []))" }, { "line": 667, "value_line": 667, "key": "locations", "target_expression": "complete_data['locations']", "value_expression": "[location]" }, { "line": 681, "value_line": 683, "key": "imported_cache_id", "target_expression": "cleaned_data.setdefault('extensions', {}).setdefault('orgbase', {})['imported_cache_id']", "value_expression": "raw_data.get('imported_cache_id')" }, { "line": 672, "value_line": 672, "key": "town", "target_expression": "location['town']", "value_expression": "raw_data.get('town')" }, { "line": 674, "value_line": 674, "key": "building", "target_expression": "location['building']", "value_expression": "raw_data.get('building')" } ] }, { "symbol": "OrgbaseCrawler.transform_offline", "path": "aggregator/orgbase_crawler.py", "line_start": 116, "line_end": 136, "inherited": false, "file_sha256": "dc4d35999fb94c47f620157a7d73a916d85c75809aa9243b48f017ea200c4d5c", "self_calls": [ "transform" ], "literal_get_calls": [ { "line": 127, "receiver": "payload", "key": "orgbase_id", "expression": "payload.get('orgbase_id')" }, { "line": 131, "receiver": "payload", "key": "raw_html", "expression": "payload.get('raw_html')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 128, "value_line": 128, "key": "orgbase_id", "target_expression": "payload['orgbase_id']", "value_expression": "orgbase_source_id(payload)" }, { "line": 131, "value_line": 131, "key": "html", "target_expression": "payload['html']", "value_expression": "payload.get('raw_html') or ''" }, { "line": 132, "value_line": 132, "key": "crawl_run_id", "target_expression": "payload['crawl_run_id']", "value_expression": "crawl_run_id" }, { "line": 133, "value_line": 133, "key": "transform_run_id", "target_expression": "payload['transform_run_id']", "value_expression": "transform_run_id" }, { "line": 134, "value_line": 134, "key": "normalized_at", "target_expression": "payload['normalized_at']", "value_expression": "normalized_at" }, { "line": 135, "value_line": 135, "key": "artifact_ids", "target_expression": "payload['artifact_ids']", "value_expression": "list(artifact_ids or [])" }, { "line": 130, "value_line": 130, "key": "imported_cache_id", "target_expression": "payload['imported_cache_id']", "value_expression": "imported_cache_id" } ] } ], "selected_rule_rows": [ { "symbol": "OrgbaseCrawler._structure_technical_data", "path": "aggregator/orgbase_crawler.py", "line": 964, "key": "Total number of stops", "value_expression": "'stop_count'" }, { "symbol": "OrgbaseCrawler._structure_technical_data", "path": "aggregator/orgbase_crawler.py", "line": 965, "key": "Key action", "value_expression": "'key_action'" }, { "symbol": "OrgbaseCrawler._process_stop_specs", "path": "aggregator/orgbase_crawler.py", "line": 1938, "value_line": 1938, "key": "mixture", "target_expression": "stop_spec_data['mixture']", "value_expression": "clean_string_by_ls_elements(mixture_detected, mixture_identifiers)" }, { "symbol": "OrgbaseCrawler._process_stop_specs", "path": "aggregator/orgbase_crawler.py", "line": 1986, "value_line": 1986, "key": "range_mapped", "target_expression": "stop_spec_data['range_mapped']", "value_expression": "convert_pipelength_to_midi(length=primary_length, fractal=primary_fractal)" }, { "symbol": "OrgbaseCrawler._process_stop_specs", "path": "aggregator/orgbase_crawler.py", "line": 1997, "value_line": 1997, "key": "transposition_sounding", "target_expression": "stop_spec_data['transposition_sounding']", "value_expression": "int(stop_spec_data['range_mapped'][0] - 36)" }, { "symbol": "OrgbaseCrawler._process_stop_specs", "path": "aggregator/orgbase_crawler.py", "line": 2003, "value_line": 2003, "key": "range", "target_expression": "split_object['range']", "value_expression": "[36, 59]" } ], "bounded_code_passages": [ { "path": "aggregator/orgbase_crawler.py", "symbol": "OrgbaseCrawler._structure_technical_data", "line_start": 961, "line_end": 969, "text": " specs = outdata[\"specifications\"]\n\n label_map = {\n \"Total number of stops\": \"stop_count\",\n \"Key action\": \"key_action\",\n \"Stop action\": \"stop_action\",\n \"Windchest(s)\": \"windchests\",\n \"Temperament\": \"temperament\",\n \"Pitch\": \"pitch\"," }, { "path": "aggregator/orgbase_crawler.py", "symbol": "OrgbaseCrawler._structure_technical_data", "line_start": 962, "line_end": 970, "text": "\n label_map = {\n \"Total number of stops\": \"stop_count\",\n \"Key action\": \"key_action\",\n \"Stop action\": \"stop_action\",\n \"Windchest(s)\": \"windchests\",\n \"Temperament\": \"temperament\",\n \"Pitch\": \"pitch\",\n \"Wind pressure\": \"wind_pressure\"," }, { "path": "aggregator/orgbase_crawler.py", "symbol": "OrgbaseCrawler._process_stop_specs", "line_start": 1935, "line_end": 1943, "text": " mixture_detected = mixt_match.group(0)\n matched_mixture_strings.append(mixture_detected)\n if mixture_detected:\n stop_spec_data[\"mixture\"] = clean_string_by_ls_elements(mixture_detected, mixture_identifiers)\n lengths, fractals, rank_matches = get_pipelength_from_string(input_str=stop)\n matched_pipelength_strings.extend(rank_matches)\n mixture_ranks_detected = True if lengths or fractals else False\n for length, fractal in zip(lengths, fractals):\n mixture_object = copy.deepcopy(self.orgbase_template[\"specifications\"][\"stops\"][0][\"mixture_ranks\"][0])" }, { "path": "aggregator/orgbase_crawler.py", "symbol": "OrgbaseCrawler._process_stop_specs", "line_start": 1983, "line_end": 1991, "text": " stop_spec_data[\"length\"] = primary_length\n stop_spec_data[\"fractal\"] = primary_fractal\n try:\n stop_spec_data[\"range_mapped\"] = convert_pipelength_to_midi(\n length=primary_length, fractal=primary_fractal\n )\n except (ValueError, ZeroDivisionError) as e:\n self.logger.warning(f\"Could not map pipelength to MIDI for '{stop}': {e}\")\n" }, { "path": "aggregator/orgbase_crawler.py", "symbol": "OrgbaseCrawler._process_stop_specs", "line_start": 1994, "line_end": 2002, "text": "\n # Identify Sound Transposition by range_mapped\n if stop_spec_data.get(\"range_mapped\"):\n stop_spec_data[\"transposition_sounding\"] = int(stop_spec_data[\"range_mapped\"][0] - 36)\n\n # Handle B/D (This logic is preserved)\n if any(\"B/D\" in substr for substr in notes):\n split_object = copy.deepcopy(self.orgbase_template[\"specifications\"][\"stops\"][0][\"splits\"][0])\n split_object[\"region\"] = \"bass\"" }, { "path": "aggregator/orgbase_crawler.py", "symbol": "OrgbaseCrawler._process_stop_specs", "line_start": 2000, "line_end": 2008, "text": " if any(\"B/D\" in substr for substr in notes):\n split_object = copy.deepcopy(self.orgbase_template[\"specifications\"][\"stops\"][0][\"splits\"][0])\n split_object[\"region\"] = \"bass\"\n split_object[\"range\"] = [36, 59]\n stop_spec_data[\"splits\"].append(split_object)\n split_object = copy.deepcopy(self.orgbase_template[\"specifications\"][\"stops\"][0][\"splits\"][0])\n split_object[\"region\"] = \"discant\"\n split_object[\"range\"] = [60, 96]\n stop_spec_data[\"splits\"].append(split_object)" } ] } }, { "catalogue_id": "S030", "source_key": "orgelarchiv_schmidt", "name": "Orgel-Archiv Schmidt", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Andreas Schmidt; Stefan Weyreder", "scope_description": "Germany, Switzerland, Austria and further country sections", "documentary_unit": "Organ profile", "source_description": "The site's history traces the family archive to Horst Schmidt in 1972 and its continuation by Andreas Schmidt from 1995. Andreas Schmidt and Stefan Weyreder established the website. The account stresses personal inspection and the identification of reliable external sources.", "source_description_references": [ "orgelarchiv_schmidt", "orgelarchiv_about", "record_orgelarchiv_schmidt" ], "interpretation": "This stated editorial practice is relevant to source criticism, but it is not an independent accuracy audit. Instrument-level credits and inspection dates are still needed when using an entry as evidence.", "rights_observation": "The website marks content as protected; no general open licence was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgelarchiv_schmidt", "orgelarchiv_about", "record_orgelarchiv_schmidt" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgelarchiv_schmidt", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Contributed organ article with labelled location fields and distinct history sections", "repeated_information": "Manual/pedal stop groups, playing aids, building history and source credits.", "date_and_state_distinctions": "Organ history, church history, inspection year and condition date are separately stated.", "interpretive_limit": "The example distinguishes planned but unbuilt stops from installed stops. Vacancies must not be counted as sounding registers.", "observations": [ "Church, denomination, place and administrative geography are labelled.", "Builders are named in the organ chronology.", "Dated construction, rebuilding and damage history.", "Stop groups explicitly include vacant positions.", "Division headings include key ranges.", "Historical action and windchest type are specified.", "Detailed controls and console fittings are described.", "A dated statement says the selected organ is unplayable.", "Photographic gallery and named contributor.", "Sources distinguish on-site inspection from published building history.", "A referenced heritage article is linked.", "Named author and inspection year are attached to the contribution." ], "uninspected_or_unfilled_material": [], "narrative": "The Mansfeld account separates organ and building histories and identifies inspection and source credits. The specification includes vacant positions for stops planned but not built. A dated statement describes the organ as unplayable. Vacancies should not enter a count of sounding registers, and the reported condition should retain its inspection date." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The adapter has a WordPress-oriented discovery route and acquires the article body with its labels, history, disposition and source sections.", "transformation": "Section boundaries are explicitly defined. Disposition text is split into versions; the first version supplies the principal specification and the remaining versions are preserved as historical specifications. Declared and parsed stop counts are stored separately, and vacant positions are retained. History is passed to a source-specific participant helper rather than silently becoming resolved people or organisations.", "limitations": "The first-version choice is an implementation convention that needs to match the article's ordering. Vacant positions are not sounding stops. Parsed totals can differ from declared historical totals, especially where several versions occur. Preserving raw history does not resolve ambiguous participant identities." }, "implementation": { "class": "OrgelArchivSchmidtCrawler", "class_line": 3603, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrgelArchivSchmidtCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orgelarchiv_schmidt", "path": "aggregator/static_html_crawler.py", "line": 3604, "class": "OrgelArchivSchmidtCrawler" }, "id_column": { "value": "orgelarchiv_schmidt_id", "path": "aggregator/static_html_crawler.py", "line": 3605, "class": "OrgelArchivSchmidtCrawler" }, "default_country": { "value": "Deutschland", "path": "aggregator/static_html_crawler.py", "line": 3608, "class": "OrgelArchivSchmidtCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://orgel-verzeichnis.de/wp-json/wp/v2/posts" ], "path": "aggregator/static_html_crawler.py", "line": 3609, "class": "OrgelArchivSchmidtCrawler" }, "detail_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 31, "class": "StaticHtmlOrganCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://orgel-verzeichnis.de", "path": "aggregator/static_html_crawler.py", "line": 3607, "class": "OrgelArchivSchmidtCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._builder_history_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 3804, "line_end": 3806, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._divisions_from_disposition", "path": "aggregator/static_html_crawler.py", "line_start": 3808, "line_end": 3823, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_is_schmidt_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 3822, "key": "name", "value_expression": "line" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._extract_schmidt_title", "path": "aggregator/static_html_crawler.py", "line_start": 3762, "line_end": 3767, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 3767, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" } ], "html_queries": [ { "line": 3763, "expression": "soup.find(['h1', 'h2'])" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 3611, "line_end": 3615, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 3612, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._is_schmidt_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 3825, "line_end": 3836, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._label_fields", "path": "aggregator/static_html_crawler.py", "line_start": 3781, "line_end": 3789, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._manual_count_from_spec", "path": "aggregator/static_html_crawler.py", "line_start": 3864, "line_end": 3870, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 202, "line_end": 211, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 205, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._reference_from_post", "path": "aggregator/static_html_crawler.py", "line_start": 3673, "line_end": 3695, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_reference_from_parts", "_split_schmidt_title" ], "literal_get_calls": [ { "line": 3675, "receiver": "post", "key": "link", "expression": "post.get('link')" }, { "line": 3674, "receiver": "post", "key": "id", "expression": "post.get('id')" }, { "line": 3688, "receiver": "post", "key": "slug", "expression": "post.get('slug')" }, { "line": 3689, "receiver": "post", "key": "date", "expression": "post.get('date')" }, { "line": 3690, "receiver": "post", "key": "modified", "expression": "post.get('modified')" }, { "line": 3691, "receiver": "post", "key": "featured_media", "expression": "post.get('featured_media')" }, { "line": 3678, "receiver": "post.get('title') or {}", "key": "rendered", "expression": "(post.get('title') or {}).get('rendered')" }, { "line": 3678, "receiver": "post", "key": "title", "expression": "post.get('title')" } ], "html_queries": [], "dictionary_entries": [ { "line": 3687, "key": "wp_id", "value_expression": "post_id" }, { "line": 3688, "key": "slug", "value_expression": "post.get('slug') or ''" }, { "line": 3689, "key": "date", "value_expression": "post.get('date') or ''" }, { "line": 3690, "key": "modified", "value_expression": "post.get('modified') or ''" }, { "line": 3691, "key": "featured_media", "value_expression": "post.get('featured_media') or ''" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._split_schmidt_title", "path": "aggregator/static_html_crawler.py", "line_start": 3769, "line_end": 3775, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler._stop_count_from_history", "path": "aggregator/static_html_crawler.py", "line_start": 3872, "line_end": 3874, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 3617, "line_end": 3671, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_optional_int", "_reference_from_post", "_save_reference", "_update_scrape_status" ], "literal_get_calls": [ { "line": 3618, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 3619, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 3620, "receiver": "kwargs", "key": "page", "expression": "kwargs.get('page')" }, { "line": 3621, "receiver": "kwargs", "key": "per_page", "expression": "kwargs.get('per_page')" }, { "line": 3621, "receiver": "self.config", "key": "per_page", "expression": "self.config.get('per_page', 100)" }, { "line": 3659, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" }, { "line": 3634, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 3630, "key": "per_page", "value_expression": "per_page" }, { "line": 3631, "key": "page", "value_expression": "page" }, { "line": 3632, "key": "_fields", "value_expression": "'id,link,slug,title,date,modified,featured_media'" } ], "subscript_assignments": [] }, { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 3697, "line_end": 3760, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_builder_history_from_text", "_clean_lines", "_divisions_from_disposition", "_extract_schmidt_title", "_first_int", "_label_fields", "_manual_count_from_spec", "_schmidt_images", "_section", "_split_disposition_versions", "_split_schmidt_title", "_stop_count_from_history" ], "literal_get_calls": [ { "line": 3698, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 3698, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 3710, "receiver": "fields", "key": "ort", "expression": "fields.get('ort')" }, { "line": 3710, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 3711, "receiver": "fields", "key": "gebäude oder kirche", "expression": "fields.get('gebäude oder kirche')" }, { "line": 3711, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 3712, "receiver": "fields", "key": "bundesland / kanton", "expression": "fields.get('bundesland / kanton')" }, { "line": 3712, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 3713, "receiver": "fields", "key": "land", "expression": "fields.get('land')" }, { "line": 3713, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 3715, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 3717, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 3751, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 3757, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 3749, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 3756, "receiver": "raw_data", "key": "orgelarchiv_schmidt_id", "expression": "raw_data.get('orgelarchiv_schmidt_id')" }, { "line": 3748, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" } ], "html_queries": [ { "line": 3700, "expression": "soup.find('article')" }, { "line": 3700, "expression": "soup.find('main')" } ], "dictionary_entries": [ { "line": 3721, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': country, 'region': region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 3731, "key": "builder_history", "value_expression": "builder_history" }, { "line": 3732, "key": "specifications", "value_expression": "{'manual_count': manual_count, 'stop_count': stop_count, 'declared_stop_count': declared_stop_count, 'parsed_stop_count': parsed_stop_count, 'raw_specification': current_spec, 'stops': stops_by_division, 'stops_by_division': stops_by_division, 'vacant_positions': [line for line in current_spec.splitlines() if re.search('\\\\bvacant\\\\b', line, flags=re.I)], 'historical_specifications': disposition_versions[1:], 'technical_rows_unstructured': [{'label': key, 'value': value} for key, value in fields.items() if key not in {'ort', 'gebäude oder kirche', 'bundesland / kanton', 'land'} and value]}" }, { "line": 3748, "key": "media", "value_expression": "{'images': self._schmidt_images(soup, raw_data.get('url') or self.base_url)}" }, { "line": 3749, "key": "links", "value_expression": "[{'link': raw_data.get('url'), 'type': 'source'}]" }, { "line": 3750, "key": "descriptions", "value_expression": "[{'type': 'organ_history', 'text': history}] if history else []" }, { "line": 3751, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', ''), 'wp_id': raw_data.get(self.id_column) or ''}" }, { "line": 3733, "key": "manual_count", "value_expression": "manual_count" }, { "line": 3734, "key": "stop_count", "value_expression": "stop_count" }, { "line": 3735, "key": "declared_stop_count", "value_expression": "declared_stop_count" }, { "line": 3736, "key": "parsed_stop_count", "value_expression": "parsed_stop_count" }, { "line": 3737, "key": "raw_specification", "value_expression": "current_spec" }, { "line": 3738, "key": "stops", "value_expression": "stops_by_division" }, { "line": 3739, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 3740, "key": "vacant_positions", "value_expression": "[line for line in current_spec.splitlines() if re.search('\\\\bvacant\\\\b', line, flags=re.I)]" }, { "line": 3741, "key": "historical_specifications", "value_expression": "disposition_versions[1:]" }, { "line": 3742, "key": "technical_rows_unstructured", "value_expression": "[{'label': key, 'value': value} for key, value in fields.items() if key not in {'ort', 'gebäude oder kirche', 'bundesland / kanton', 'land'} and value]" }, { "line": 3748, "key": "images", "value_expression": "self._schmidt_images(soup, raw_data.get('url') or self.base_url)" }, { "line": 3751, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 3751, "key": "wp_id", "value_expression": "raw_data.get(self.id_column) or ''" }, { "line": 3723, "key": "status", "value_expression": "'current_location'" }, { "line": 3724, "key": "country", "value_expression": "country" }, { "line": 3725, "key": "region", "value_expression": "region" }, { "line": 3726, "key": "town", "value_expression": "town" }, { "line": 3727, "key": "building", "value_expression": "building" }, { "line": 3728, "key": "raw_str", "value_expression": "title" }, { "line": 3749, "key": "link", "value_expression": "raw_data.get('url')" }, { "line": 3749, "key": "type", "value_expression": "'source'" }, { "line": 3743, "key": "label", "value_expression": "key" }, { "line": 3743, "key": "value", "value_expression": "value" }, { "line": 3750, "key": "type", "value_expression": "'organ_history'" }, { "line": 3750, "key": "text", "value_expression": "history" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3734, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3733, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3739, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3741, "key": "historical_specifications", "value_expression": "disposition_versions[1:]" }, { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3737, "key": "raw_specification", "value_expression": "current_spec" }, { "symbol": "OrgelArchivSchmidtCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3724, "key": "country", "value_expression": "country" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelArchivSchmidtCrawler.transform", "line_start": 3731, "line_end": 3739, "text": " \"builder_history\": builder_history,\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"declared_stop_count\": declared_stop_count,\n \"parsed_stop_count\": parsed_stop_count,\n \"raw_specification\": current_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelArchivSchmidtCrawler.transform", "line_start": 3730, "line_end": 3738, "text": " ],\n \"builder_history\": builder_history,\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"declared_stop_count\": declared_stop_count,\n \"parsed_stop_count\": parsed_stop_count,\n \"raw_specification\": current_spec,\n \"stops\": stops_by_division," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelArchivSchmidtCrawler.transform", "line_start": 3736, "line_end": 3744, "text": " \"parsed_stop_count\": parsed_stop_count,\n \"raw_specification\": current_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"vacant_positions\": [line for line in current_spec.splitlines() if re.search(r\"\\bvacant\\b\", line, flags=re.I)],\n \"historical_specifications\": disposition_versions[1:],\n \"technical_rows_unstructured\": [\n {\"label\": key, \"value\": value}\n for key, value in fields.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelArchivSchmidtCrawler.transform", "line_start": 3738, "line_end": 3746, "text": " \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"vacant_positions\": [line for line in current_spec.splitlines() if re.search(r\"\\bvacant\\b\", line, flags=re.I)],\n \"historical_specifications\": disposition_versions[1:],\n \"technical_rows_unstructured\": [\n {\"label\": key, \"value\": value}\n for key, value in fields.items()\n if key not in {\"ort\", \"gebäude oder kirche\", \"bundesland / kanton\", \"land\"} and value\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelArchivSchmidtCrawler.transform", "line_start": 3734, "line_end": 3742, "text": " \"stop_count\": stop_count,\n \"declared_stop_count\": declared_stop_count,\n \"parsed_stop_count\": parsed_stop_count,\n \"raw_specification\": current_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"vacant_positions\": [line for line in current_spec.splitlines() if re.search(r\"\\bvacant\\b\", line, flags=re.I)],\n \"historical_specifications\": disposition_versions[1:],\n \"technical_rows_unstructured\": [" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelArchivSchmidtCrawler.transform", "line_start": 3721, "line_end": 3729, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": country,\n \"region\": region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" } ] } }, { "catalogue_id": "S031", "source_key": "orgeldb_bayern", "name": "Orgeldatenbank Bayern", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Michael Bernhard (editor); Gesellschaft für Bayerische Musikgeschichte e.V. (original publisher)", "scope_description": "Bavaria; version 5, 2009", "documentary_unit": "Historical database record, including successive descriptions", "source_description": "The portal serves an authorised copy of the 2009 database, described as exceeding 30,000 records with roughly 10,000 stoplists. It adds location matching using Wikipedia, Wikidata and OpenStreetMap and explicitly warns that some instruments labelled extant no longer survive in that form.", "source_description_references": [ "orgeldb_bayern", "record_orgeldb_bayern" ], "interpretation": "Database date and later geolocation are separate provenance layers. The record total cannot be read as a count of present organs. Treat apparent current-state statements as historical until corroborated.", "rights_observation": "Permission to host the copy is stated; this is not a blanket permission for third-party redistribution.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgeldb_bayern", "record_orgeldb_bayern" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgeldb_bayern", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "CGI database record combining historical text fields with later geographic links", "repeated_information": "Compact work statement, manual/register summary and inline division stoplists.", "date_and_state_distinctions": "Version 5 (2009) is the underlying documentary state; geographic enrichment is a separate layer.", "interpretive_limit": "The example URL identifier and displayed source record number differ. Neither should silently replace the other.", "observations": [ "Postal place, district, administrative region, church and street address.", "Coordinates and Wikidata/OpenStreetMap references are displayed.", "Named builder associated with a work entry.", "New-build year and retained/extant status accompany the record.", "Stops are written inline by division.", "Manual divisions and a manual/register summary are present.", "Windchest and mechanical-action description.", "Case style and couplers/playing aids are stated.", "Historical retained/extant label is displayed.", "Bibliographic source with year and page plus source record number.", "Wikidata, Wikipedia and map links accompany the record.", "The source version is identified as 2009." ], "uninspected_or_unfilled_material": [], "narrative": "The selected record combines a compact work statement and inline division specification with bibliographic and geographical links. The number displayed as the source record number differs from the identifier in the URL. Both should be retained with their roles rather than silently substituting one for the other. Later geographic enrichment does not redate the underlying 2009 database." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The adapter contains distinct paths for HTML detail pages and Access MDB input. The native record number, rather than a inferred physical-organ identity, supplies its primary documentary key.", "transformation": "The HTML branch reads preformatted lines, location text, labelled builder and event information, technical counts and stop groups. It also extracts coordinates and a Wikidata identifier when present. The MDB branch is selected by a source-format marker and must be evaluated against its own fields. The HTML branch preserves status, source reference and record number among unstructured technical rows.", "limitations": "The two input formats should not be assumed to expose identical fields or versions. In the HTML branch, the derived description is truncated to 6,000 characters, while the original HTML is passed to the snapshot normalizer. A source number, Wikidata link and cross-source identity assertion have different evidential roles." }, "implementation": { "class": "OrgeldbBayernCrawler", "class_line": 1411, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrgeldbBayernCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orgeldb_bayern", "path": "aggregator/static_html_crawler.py", "line": 1412, "class": "OrgeldbBayernCrawler" }, "id_column": { "value": "orgeldb_bayern_id", "path": "aggregator/static_html_crawler.py", "line": 1413, "class": "OrgeldbBayernCrawler" }, "default_country": { "value": "Deutschland", "path": "aggregator/static_html_crawler.py", "line": 1417, "class": "OrgeldbBayernCrawler" }, "default_region": { "value": "Bayern", "path": "aggregator/static_html_crawler.py", "line": 1418, "class": "OrgeldbBayernCrawler" }, "start_urls": { "value": [ "https://orgeldb.de/cgi-bin/lkr" ], "path": "aggregator/static_html_crawler.py", "line": 1419, "class": "OrgeldbBayernCrawler" }, "detail_url_patterns": { "value": [ "dispo?satz=" ], "path": "aggregator/static_html_crawler.py", "line": 1420, "class": "OrgeldbBayernCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://orgeldb.de", "path": "aggregator/static_html_crawler.py", "line": 1416, "class": "OrgeldbBayernCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._crawl_mdb", "path": "aggregator/static_html_crawler.py", "line_start": 1487, "line_end": 1517, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_iter_mdb_rows", "_mdb_row_is_empty", "_optional_int", "_reference_from_mdb_row", "_save_reference", "_update_scrape_status" ], "literal_get_calls": [ { "line": 1488, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 1489, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 1489, "receiver": "kwargs", "key": "limit", "expression": "kwargs.get('limit')" }, { "line": 1500, "receiver": "ref", "key": "_raw_snapshot", "expression": "ref.get('_raw_snapshot', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._detail_title", "path": "aggregator/static_html_crawler.py", "line_start": 1838, "line_end": 1842, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 1842, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" } ], "html_queries": [ { "line": 1839, "expression": "soup.find('h3')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._event_from_lines", "path": "aggregator/static_html_crawler.py", "line_start": 1866, "line_end": 1876, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 1871, "key": "activity", "value_expression": "self._clean_text(match.group('activity')).lower()" }, { "line": 1872, "key": "year", "value_expression": "match.group('year')" }, { "line": 1873, "key": "status", "value_expression": "self._clean_text(match.group('status') or '')" }, { "line": 1874, "key": "raw", "value_expression": "line" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 1940, "line_end": 1959, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 1942, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 1958, "key": "url", "value_expression": "full" }, { "line": 1958, "key": "link", "value_expression": "full" }, { "line": 1958, "key": "label", "value_expression": "label" }, { "line": 1958, "key": "type", "value_expression": "link_type" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._location_line", "path": "aggregator/static_html_crawler.py", "line_start": 1844, "line_end": 1853, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 1845, "expression": "soup.find('h3')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._manuals_stops", "path": "aggregator/static_html_crawler.py", "line_start": 1886, "line_end": 1888, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_value_after_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._mdb_event", "path": "aggregator/static_html_crawler.py", "line_start": 1791, "line_end": 1803, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 1795, "receiver": "row", "key": "BEMERKUNG", "expression": "row.get('BEMERKUNG')" }, { "line": 1792, "receiver": "row", "key": "T", "expression": "row.get('T')" } ], "html_queries": [], "dictionary_entries": [ { "line": 1793, "key": "N", "value_expression": "'construction'" }, { "line": 1793, "key": "R", "value_expression": "'repair'" }, { "line": 1793, "key": "U", "value_expression": "'modification'" }, { "line": 1793, "key": "E", "value_expression": "'extension'" }, { "line": 1797, "key": "type", "value_expression": "event_type" }, { "line": 1798, "key": "date", "value_expression": "parsed_year" }, { "line": 1799, "key": "dates", "value_expression": "[parsed_year] if parsed_year else []" }, { "line": 1800, "key": "description", "value_expression": "'; '.join((part for part in description_parts if part))" }, { "line": 1801, "key": "raw", "value_expression": "raw_year" }, { "line": 1802, "key": "source_activity_code", "value_expression": "activity_code" } ], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._mdb_literature", "path": "aggregator/static_html_crawler.py", "line_start": 1805, "line_end": 1820, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 1813, "key": "citation", "value_expression": "citation" }, { "line": 1814, "key": "slot", "value_expression": "index" }, { "line": 1815, "key": "has_disposition", "value_expression": "bool(row.get(f'DISP{index}'))" }, { "line": 1816, "key": "has_image", "value_expression": "bool(row.get(f'ABB{index}'))" }, { "line": 1817, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._mdb_location", "path": "aggregator/static_html_crawler.py", "line_start": 1770, "line_end": 1789, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 1771, "receiver": "row", "key": "ADRESSE2", "expression": "row.get('ADRESSE2')" }, { "line": 1772, "receiver": "row", "key": "POST", "expression": "row.get('POST')" }, { "line": 1772, "receiver": "row", "key": "POST", "expression": "row.get('POST')" }, { "line": 1777, "receiver": "row", "key": "ERHALTEN", "expression": "row.get('ERHALTEN')" }, { "line": 1780, "receiver": "row", "key": "RBZ", "expression": "row.get('RBZ')" }, { "line": 1781, "receiver": "row", "key": "LKR", "expression": "row.get('LKR')" }, { "line": 1782, "receiver": "row", "key": "GDE", "expression": "row.get('GDE')" }, { "line": 1786, "receiver": "row", "key": "BEZ", "expression": "row.get('BEZ')" }, { "line": 1787, "receiver": "row", "key": "KL", "expression": "row.get('KL')" }, { "line": 1783, "receiver": "row", "key": "ORT", "expression": "row.get('ORT')" }, { "line": 1783, "receiver": "row", "key": "GDE", "expression": "row.get('GDE')" }, { "line": 1783, "receiver": "row", "key": "ADRESSE3", "expression": "row.get('ADRESSE3')" }, { "line": 1784, "receiver": "row", "key": "KIRCHE", "expression": "row.get('KIRCHE')" }, { "line": 1784, "receiver": "row", "key": "ADRESSE1", "expression": "row.get('ADRESSE1')" }, { "line": 1773, "receiver": "row", "key": "ADRESSE3", "expression": "row.get('ADRESSE3')" } ], "html_queries": [], "dictionary_entries": [ { "line": 1777, "key": "status", "value_expression": "'current_location' if row.get('ERHALTEN') else 'historical_location'" }, { "line": 1778, "key": "country", "value_expression": "self.default_country" }, { "line": 1779, "key": "region", "value_expression": "self.default_region" }, { "line": 1780, "key": "administrative_region", "value_expression": "self._clean_text(row.get('RBZ'))" }, { "line": 1781, "key": "county", "value_expression": "self._clean_text(row.get('LKR'))" }, { "line": 1782, "key": "municipality", "value_expression": "self._clean_text(row.get('GDE'))" }, { "line": 1783, "key": "town", "value_expression": "self._clean_text(row.get('ORT') or row.get('GDE') or row.get('ADRESSE3'))" }, { "line": 1784, "key": "building", "value_expression": "self._clean_text(row.get('KIRCHE') or row.get('ADRESSE1'))" }, { "line": 1785, "key": "address", "value_expression": "', '.join((part for part in address_parts if part))" }, { "line": 1786, "key": "place_type", "value_expression": "self._clean_text(row.get('BEZ'))" }, { "line": 1787, "key": "church_type", "value_expression": "self._clean_text(row.get('KL'))" }, { "line": 1788, "key": "raw_str", "value_expression": "title" } ], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._mdb_title", "path": "aggregator/static_html_crawler.py", "line_start": 1765, "line_end": 1768, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 1766, "receiver": "row", "key": "ORT", "expression": "row.get('ORT')" }, { "line": 1766, "receiver": "row", "key": "GDE", "expression": "row.get('GDE')" }, { "line": 1766, "receiver": "row", "key": "ADRESSE3", "expression": "row.get('ADRESSE3')" }, { "line": 1767, "receiver": "row", "key": "KIRCHE", "expression": "row.get('KIRCHE')" }, { "line": 1767, "receiver": "row", "key": "ADRESSE1", "expression": "row.get('ADRESSE1')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 1582, "line_end": 1612, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_clean_year", "_dedupe_refs", "_reference_from_parts", "_satz_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 1584, "expression": "soup.find('title')" }, { "line": 1588, "expression": "soup.find_all('tr')" }, { "line": 1592, "expression": "row.find('a', href=re.compile('dispo\\\\?satz=\\\\d+'))" }, { "line": 1589, "expression": "row.find_all(['td', 'th'], recursive=False)" } ], "dictionary_entries": [ { "line": 1608, "key": "county", "value_expression": "county" }, { "line": 1608, "key": "row", "value_expression": "cells" } ], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._parse_location_line", "path": "aggregator/static_html_crawler.py", "line_start": 1855, "line_end": 1864, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 1864, "key": "town", "value_expression": "town or parsed_town or (parts[0] if parts else '')" }, { "line": 1864, "key": "building", "value_expression": "building" }, { "line": 1864, "key": "address", "value_expression": "address" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "OrgeldbBayernCrawler._reference_from_mdb_row", "path": "aggregator/static_html_crawler.py", "line_start": 1533, "line_end": 1557, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_clean_year", "_reference_from_parts" ], "literal_get_calls": [ { "line": 1534, "receiver": "row", "key": "SATZ", "expression": "row.get('SATZ')" }, { "line": 1535, "receiver": "row", "key": "ORT", "expression": "row.get('ORT')" }, { "line": 1535, "receiver": "row", "key": "GDE", "expression": "row.get('GDE')" }, { "line": 1535, "receiver": "row", "key": "ADRESSE3", "expression": "row.get('ADRESSE3')" }, { "line": 1536, "receiver": "row", "key": "KIRCHE", "expression": "row.get('KIRCHE')" }, { "line": 1536, "receiver": "row", "key": "ADRESSE1", "expression": "row.get('ADRESSE1')" }, { "line": 1552, "receiver": "row", "key": "BAUER", "expression": "row.get('BAUER')" }, { "line": 1554, "receiver": "row", "key": "M", "expression": "row.get('M')" }, { "line": 1554, "receiver": "row", "key": "M", "expression": "row.get('M')" }, { "line": 1555, "receiver": "row", "key": "REG", "expression": "row.get('REG')" }, { "line": 1555, "receiver": "row", "key": "REG", "expression": "row.get('REG')" }, { "line": 1553, "receiver": "row", "key": "JAHR", "expression": "row.get('JAHR')" } ], "html_queries": [], "dictionary_entries": [ { "line": 1540, "key": "source_format", "value_expression": "'access_mdb'" }, { "line": 1541, "key": "mdb_path", "value_expression": "mdb_path" }, { "line": 1542, "key": "table", "value_expression": "'Bydaten'" }, { "line": 1543, "key": "row", "value_expression": "row" }, { "line": 1557, "key": "_raw_snapshot", "value_expression": "raw_snapshot" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._source_reference", "path": "aggregator/static_html_crawler.py", "line_start": 1920, "line_end": 1921, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_value_after_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._spec_from_lines", "path": "aggregator/static_html_crawler.py", "line_start": 1890, "line_end": 1918, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 1912, "key": "name", "value_expression": "stop" }, { "line": 1912, "key": "raw_str", "value_expression": "stop" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._transform_mdb", "path": "aggregator/static_html_crawler.py", "line_start": 1702, "line_end": 1763, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_clean_dispo", "_clean_text", "_clean_year", "_first_int", "_mdb_description", "_mdb_event", "_mdb_literature", "_mdb_location", "_mdb_title", "_spec_from_lines" ], "literal_get_calls": [ { "line": 1703, "receiver": "raw_payload", "key": "row", "expression": "raw_payload.get('row')" }, { "line": 1705, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 1706, "receiver": "row", "key": "DISPO", "expression": "row.get('DISPO')" }, { "line": 1709, "receiver": "row", "key": "JAHR", "expression": "row.get('JAHR')" }, { "line": 1711, "receiver": "row", "key": "BAUER", "expression": "row.get('BAUER')" }, { "line": 1704, "receiver": "row", "key": "SATZ", "expression": "row.get('SATZ')" }, { "line": 1720, "receiver": "row", "key": "M", "expression": "row.get('M')" }, { "line": 1721, "receiver": "row", "key": "REG", "expression": "row.get('REG')" }, { "line": 1746, "receiver": "raw_payload", "key": "mdb_path", "expression": "raw_payload.get('mdb_path', '')" }, { "line": 1747, "receiver": "raw_payload", "key": "table", "expression": "raw_payload.get('table', 'Bydaten')" }, { "line": 1760, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 1724, "receiver": "stop", "key": "raw_str", "expression": "stop.get('raw_str', stop['name'])" }, { "line": 1732, "receiver": "row", "key": "ERHALTEN", "expression": "row.get('ERHALTEN')" }, { "line": 1733, "receiver": "row", "key": "DO", "expression": "row.get('DO')" }, { "line": 1734, "receiver": "row", "key": "DP", "expression": "row.get('DP')" }, { "line": 1735, "receiver": "row", "key": "DISPOJA", "expression": "row.get('DISPOJA')" }, { "line": 1749, "receiver": "row", "key": "T", "expression": "row.get('T')" }, { "line": 1731, "receiver": "row", "key": "T", "expression": "row.get('T')" }, { "line": 1737, "receiver": "row", "key": "BEMERKUNG", "expression": "row.get('BEMERKUNG')" }, { "line": 1737, "receiver": "row", "key": "MEMO", "expression": "row.get('MEMO')" } ], "html_queries": [], "dictionary_entries": [ { "line": 1716, "key": "locations", "value_expression": "[location]" }, { "line": 1717, "key": "involved", "value_expression": "self._build_involved(builder, parsed_year)" }, { "line": 1718, "key": "events", "value_expression": "[event] if event else []" }, { "line": 1739, "key": "literature", "value_expression": "literature" }, { "line": 1740, "key": "links", "value_expression": "[]" }, { "line": 1741, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': self._mdb_description(row, title, literature)}]" }, { "line": 1742, "key": "other_identifiers", "value_expression": "[{'type': self.id_column, 'value': primary_id}]" }, { "line": 1743, "key": "extensions", "value_expression": "{self.source_key: {'source_format': 'access_mdb', 'mdb_path': raw_payload.get('mdb_path', ''), 'table': raw_payload.get('table', 'Bydaten'), 'raw_row': row, 'activity_code': self._clean_text(row.get('T')), 'year_raw': raw_year, 'literature_slots': literature}}" }, { "line": 1720, "key": "manual_count", "value_expression": "self._first_int(row.get('M'))" }, { "line": 1721, "key": "stop_count", "value_expression": "self._first_int(row.get('REG'))" }, { "line": 1722, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 1723, "key": "stops", "value_expression": "[{'name': stop['name'], 'division': division, 'raw_str': stop.get('raw_str', stop['name'])} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 1728, "key": "raw_spec", "value_expression": "parsed_spec or raw_spec" }, { "line": 1729, "key": "technical_rows_unstructured", "value_expression": "[{'label': 'Satz', 'value': primary_id}, {'label': 'Typ', 'value': self._clean_text(row.get('T'))}, {'label': 'Erhalten', 'value': row.get('ERHALTEN')}, {'label': 'Disposition Online', 'value': row.get('DO')}, {'label': 'Disposition Printed', 'value': row.get('DP')}, {'label': 'Dispoja', 'value': row.get('DISPOJA')}]" }, { "line": 1737, "key": "notes", "value_expression": "[item for item in (self._clean_text(row.get('BEMERKUNG')), self._clean_text(row.get('MEMO'))) if item]" }, { "line": 1741, "key": "type", "value_expression": "'source_text'" }, { "line": 1741, "key": "text", "value_expression": "self._mdb_description(row, title, literature)" }, { "line": 1742, "key": "type", "value_expression": "self.id_column" }, { "line": 1742, "key": "value", "value_expression": "primary_id" }, { "line": 1745, "key": "source_format", "value_expression": "'access_mdb'" }, { "line": 1746, "key": "mdb_path", "value_expression": "raw_payload.get('mdb_path', '')" }, { "line": 1747, "key": "table", "value_expression": "raw_payload.get('table', 'Bydaten')" }, { "line": 1748, "key": "raw_row", "value_expression": "row" }, { "line": 1749, "key": "activity_code", "value_expression": "self._clean_text(row.get('T'))" }, { "line": 1750, "key": "year_raw", "value_expression": "raw_year" }, { "line": 1751, "key": "literature_slots", "value_expression": "literature" }, { "line": 1724, "key": "name", "value_expression": "stop['name']" }, { "line": 1724, "key": "division", "value_expression": "division" }, { "line": 1724, "key": "raw_str", "value_expression": "stop.get('raw_str', stop['name'])" }, { "line": 1730, "key": "label", "value_expression": "'Satz'" }, { "line": 1730, "key": "value", "value_expression": "primary_id" }, { "line": 1731, "key": "label", "value_expression": "'Typ'" }, { "line": 1731, "key": "value", "value_expression": "self._clean_text(row.get('T'))" }, { "line": 1732, "key": "label", "value_expression": "'Erhalten'" }, { "line": 1732, "key": "value", "value_expression": "row.get('ERHALTEN')" }, { "line": 1733, "key": "label", "value_expression": "'Disposition Online'" }, { "line": 1733, "key": "value", "value_expression": "row.get('DO')" }, { "line": 1734, "key": "label", "value_expression": "'Disposition Printed'" }, { "line": 1734, "key": "value", "value_expression": "row.get('DP')" }, { "line": 1735, "key": "label", "value_expression": "'Dispoja'" }, { "line": 1735, "key": "value", "value_expression": "row.get('DISPOJA')" } ], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler._value_after_label", "path": "aggregator/static_html_crawler.py", "line_start": 1878, "line_end": 1884, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 1422, "line_end": 1485, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_county_names", "_crawl_mdb", "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_post_county", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 1424, "receiver": "kwargs", "key": "mdb_path", "expression": "kwargs.get('mdb_path')" }, { "line": 1425, "receiver": "kwargs", "key": "access_mdb_path", "expression": "kwargs.get('access_mdb_path')" }, { "line": 1426, "receiver": "self.config", "key": "mdb_path", "expression": "self.config.get('mdb_path')" }, { "line": 1435, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 1436, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 1437, "receiver": "kwargs", "key": "counties", "expression": "kwargs.get('counties')" }, { "line": 1437, "receiver": "self.config", "key": "counties", "expression": "self.config.get('counties')" }, { "line": 1437, "receiver": "self.config", "key": "default_counties", "expression": "self.config.get('default_counties')" }, { "line": 1474, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgeldbBayernCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 1614, "line_end": 1700, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_clean_text", "_coordinates", "_detail_title", "_display_satz", "_event_from_lines", "_extract_links", "_first_int", "_location_line", "_main_text", "_manuals_stops", "_parse_location_line", "_safe_json_loads", "_satz_from_url", "_source_reference", "_spec_from_lines", "_split_title", "_transform_mdb", "_value_after_label", "_wikidata_id" ], "literal_get_calls": [ { "line": 1615, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" }, { "line": 1616, "receiver": "raw_payload", "key": "source_format", "expression": "raw_payload.get('source_format')" }, { "line": 1619, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 1681, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 1683, "receiver": "self._safe_json_loads(raw_data.get('raw_data_json'), {})", "key": "county", "expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {}).get('county', '')" }, { "line": 1697, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 1633, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 1647, "receiver": "location", "key": "address", "expression": "location.get('address', '')" }, { "line": 1653, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 1653, "receiver": "event", "key": "year", "expression": "event.get('year', '')" }, { "line": 1653, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 1660, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 1677, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 1645, "receiver": "location", "key": "town", "expression": "location.get('town')" }, { "line": 1645, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 1646, "receiver": "location", "key": "building", "expression": "location.get('building')" }, { "line": 1646, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 1658, "receiver": "event", "key": "raw", "expression": "event.get('raw', '')" }, { "line": 1662, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 1663, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 1666, "receiver": "stop", "key": "raw_str", "expression": "stop.get('raw_str', stop['name'])" }, { "line": 1672, "receiver": "event", "key": "status", "expression": "event.get('status', '')" }, { "line": 1656, "receiver": "event", "key": "activity", "expression": "event.get('activity')" }, { "line": 1657, "receiver": "event", "key": "year", "expression": "event.get('year')" }, { "line": 1657, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 1683, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" } ], "html_queries": [ { "line": 1624, "expression": "soup.find_all('pre')" } ], "dictionary_entries": [ { "line": 1640, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': self.default_country, 'region': self.default_region, 'town': location.get('town') or raw_data.get('town') or town, 'building': location.get('building') or raw_data.get('building') or building, 'address': location.get('address', ''), 'latitude': latitude, 'longitude': longitude, 'raw_str': location_line or title}]" }, { "line": 1653, "key": "involved", "value_expression": "self._build_involved(builder or raw_data.get('builder'), event.get('year', '') or raw_data.get('year'))" }, { "line": 1654, "key": "events", "value_expression": "[{'type': event.get('activity') or 'construction', 'date': event.get('year') or raw_data.get('year') or '', 'description': event.get('raw', '')}] if event or raw_data.get('year') else []" }, { "line": 1661, "key": "specifications", "value_expression": "{'manual_count': self._first_int(manual_count or raw_data.get('manual_count')), 'stop_count': self._first_int(stop_count or raw_data.get('stop_count')), 'stops_by_division': stops_by_division, 'stops': [{'name': stop['name'], 'division': division, 'raw_str': stop.get('raw_str', stop['name'])} for division, stops in stops_by_division.items() for stop in stops], 'raw_spec': raw_spec, 'technical_rows_unstructured': [{'label': 'Status', 'value': event.get('status', '')}, {'label': 'Quelle', 'value': self._source_reference(pre_lines)}, {'label': 'Satz', 'value': self._display_satz(pre_lines)}]}" }, { "line": 1677, "key": "links", "value_expression": "self._extract_links(soup, raw_data.get('url') or '')" }, { "line": 1678, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': text[:6000]}] if text else []" }, { "line": 1679, "key": "other_identifiers", "value_expression": "other_identifiers" }, { "line": 1680, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', ''), 'location_line': location_line, 'county': self._safe_json_loads(raw_data.get('raw_data_json'), {}).get('county', '')}" }, { "line": 1685, "key": "extensions", "value_expression": "{self.source_key: {'pre_lines': pre_lines, 'wikidata': wikidata, 'coordinates': {'lat': latitude, 'lon': longitude} if latitude or longitude else {}}}" }, { "line": 1634, "key": "type", "value_expression": "self.id_column" }, { "line": 1634, "key": "value", "value_expression": "primary_id" }, { "line": 1662, "key": "manual_count", "value_expression": "self._first_int(manual_count or raw_data.get('manual_count'))" }, { "line": 1663, "key": "stop_count", "value_expression": "self._first_int(stop_count or raw_data.get('stop_count'))" }, { "line": 1664, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 1665, "key": "stops", "value_expression": "[{'name': stop['name'], 'division': division, 'raw_str': stop.get('raw_str', stop['name'])} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 1670, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 1671, "key": "technical_rows_unstructured", "value_expression": "[{'label': 'Status', 'value': event.get('status', '')}, {'label': 'Quelle', 'value': self._source_reference(pre_lines)}, {'label': 'Satz', 'value': self._display_satz(pre_lines)}]" }, { "line": 1681, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 1682, "key": "location_line", "value_expression": "location_line" }, { "line": 1683, "key": "county", "value_expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {}).get('county', '')" }, { "line": 1637, "key": "type", "value_expression": "'wikidata'" }, { "line": 1637, "key": "value", "value_expression": "wikidata" }, { "line": 1642, "key": "status", "value_expression": "'current_location'" }, { "line": 1643, "key": "country", "value_expression": "self.default_country" }, { "line": 1644, "key": "region", "value_expression": "self.default_region" }, { "line": 1645, "key": "town", "value_expression": "location.get('town') or raw_data.get('town') or town" }, { "line": 1646, "key": "building", "value_expression": "location.get('building') or raw_data.get('building') or building" }, { "line": 1647, "key": "address", "value_expression": "location.get('address', '')" }, { "line": 1648, "key": "latitude", "value_expression": "latitude" }, { "line": 1649, "key": "longitude", "value_expression": "longitude" }, { "line": 1650, "key": "raw_str", "value_expression": "location_line or title" }, { "line": 1687, "key": "pre_lines", "value_expression": "pre_lines" }, { "line": 1688, "key": "wikidata", "value_expression": "wikidata" }, { "line": 1689, "key": "coordinates", "value_expression": "{'lat': latitude, 'lon': longitude} if latitude or longitude else {}" }, { "line": 1656, "key": "type", "value_expression": "event.get('activity') or 'construction'" }, { "line": 1657, "key": "date", "value_expression": "event.get('year') or raw_data.get('year') or ''" }, { "line": 1658, "key": "description", "value_expression": "event.get('raw', '')" }, { "line": 1666, "key": "name", "value_expression": "stop['name']" }, { "line": 1666, "key": "division", "value_expression": "division" }, { "line": 1666, "key": "raw_str", "value_expression": "stop.get('raw_str', stop['name'])" }, { "line": 1672, "key": "label", "value_expression": "'Status'" }, { "line": 1672, "key": "value", "value_expression": "event.get('status', '')" }, { "line": 1673, "key": "label", "value_expression": "'Quelle'" }, { "line": 1673, "key": "value", "value_expression": "self._source_reference(pre_lines)" }, { "line": 1674, "key": "label", "value_expression": "'Satz'" }, { "line": 1674, "key": "value", "value_expression": "self._display_satz(pre_lines)" }, { "line": 1678, "key": "type", "value_expression": "'source_text'" }, { "line": 1678, "key": "text", "value_expression": "text[:6000]" }, { "line": 1689, "key": "lat", "value_expression": "latitude" }, { "line": 1689, "key": "lon", "value_expression": "longitude" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "OrgeldbBayernCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1663, "key": "stop_count", "value_expression": "self._first_int(stop_count or raw_data.get('stop_count'))" }, { "symbol": "OrgeldbBayernCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1662, "key": "manual_count", "value_expression": "self._first_int(manual_count or raw_data.get('manual_count'))" }, { "symbol": "OrgeldbBayernCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1664, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "OrgeldbBayernCrawler._mdb_event", "path": "aggregator/static_html_crawler.py", "line": 1799, "key": "dates", "value_expression": "[parsed_year] if parsed_year else []" }, { "symbol": "OrgeldbBayernCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1643, "key": "country", "value_expression": "self.default_country" }, { "symbol": "OrgeldbBayernCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1650, "key": "raw_str", "value_expression": "location_line or title" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OrgeldbBayernCrawler.transform", "line_start": 1660, "line_end": 1668, "text": " ] if event or raw_data.get(\"year\") else [],\n \"specifications\": {\n \"manual_count\": self._first_int(manual_count or raw_data.get(\"manual_count\")),\n \"stop_count\": self._first_int(stop_count or raw_data.get(\"stop_count\")),\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop[\"name\"], \"division\": division, \"raw_str\": stop.get(\"raw_str\", stop[\"name\"])}\n for division, stops in stops_by_division.items()\n for stop in stops" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgeldbBayernCrawler.transform", "line_start": 1659, "line_end": 1667, "text": " }\n ] if event or raw_data.get(\"year\") else [],\n \"specifications\": {\n \"manual_count\": self._first_int(manual_count or raw_data.get(\"manual_count\")),\n \"stop_count\": self._first_int(stop_count or raw_data.get(\"stop_count\")),\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop[\"name\"], \"division\": division, \"raw_str\": stop.get(\"raw_str\", stop[\"name\"])}\n for division, stops in stops_by_division.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgeldbBayernCrawler.transform", "line_start": 1661, "line_end": 1669, "text": " \"specifications\": {\n \"manual_count\": self._first_int(manual_count or raw_data.get(\"manual_count\")),\n \"stop_count\": self._first_int(stop_count or raw_data.get(\"stop_count\")),\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop[\"name\"], \"division\": division, \"raw_str\": stop.get(\"raw_str\", stop[\"name\"])}\n for division, stops in stops_by_division.items()\n for stop in stops\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgeldbBayernCrawler._mdb_event", "line_start": 1796, "line_end": 1803, "text": " return {\n \"type\": event_type,\n \"date\": parsed_year,\n \"dates\": [parsed_year] if parsed_year else [],\n \"description\": \"; \".join(part for part in description_parts if part),\n \"raw\": raw_year,\n \"source_activity_code\": activity_code,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgeldbBayernCrawler.transform", "line_start": 1640, "line_end": 1648, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": self.default_country,\n \"region\": self.default_region,\n \"town\": location.get(\"town\") or raw_data.get(\"town\") or town,\n \"building\": location.get(\"building\") or raw_data.get(\"building\") or building,\n \"address\": location.get(\"address\", \"\"),\n \"latitude\": latitude," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgeldbBayernCrawler.transform", "line_start": 1647, "line_end": 1655, "text": " \"address\": location.get(\"address\", \"\"),\n \"latitude\": latitude,\n \"longitude\": longitude,\n \"raw_str\": location_line or title,\n }\n ],\n \"involved\": self._build_involved(builder or raw_data.get(\"builder\"), event.get(\"year\", \"\") or raw_data.get(\"year\")),\n \"events\": [\n {" } ] } }, { "catalogue_id": "S032", "source_key": "orgeldoc", "name": "Orgeldokumentationszentrum HSLU", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Hochschule Luzern; Zentral- und Hochschulbibliothek Luzern", "scope_description": "Swiss organ culture and associated archival collections", "documentary_unit": "Organ record linked to institutional documentation", "source_description": "HSLU dates the Orgeldokumentationszentrum's activity as a competence centre to 2007-2025. Following Marco Brandazza's retirement in September 2025, the university maintains access to the organ database and catalogued archives. Its statement says that funding currently prevents additional inventory and publication projects.", "source_description_references": [ "orgeldoc", "hslu_status", "record_orgeldoc" ], "interpretation": "Institutional continuity of access must be distinguished from continued expansion. The physical archives, library holdings and online organ database are related but different research objects. Cite the relevant catalogue or record for individual holdings.", "rights_observation": "Continued access is promised; a general redistribution licence for the database and archives was not established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgeldoc", "hslu_status", "record_orgeldoc" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgeldoc", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "JSON detail endpoint with nested location, builder, organ and specification objects", "repeated_information": "Specification array contains divisions; each division contains register entries and comments.", "date_and_state_distinctions": "Work year and last-change date are distinct keys.", "interpretive_limit": "Some keys have null values. Register comments preserve materials, construction and compass-dependent exceptions.", "observations": [ "Building, building part, locality and canton are populated.", "Builder, measure and year are separate attributes.", "New-build activity and year are populated.", "Register objects are nested under division objects.", "Manual and pedal range strings are supplied.", "Separate key and stop actions.", "Pitch with temperature and tuning-system fields.", "Bellows arrangement and wind-pressure value.", "Playing aids and a detailed comment are populated.", "A last-change date is supplied." ], "uninspected_or_unfilled_material": [], "narrative": "Detail 17202 is returned as nested JSON for location, builder, organ and specification. Divisions contain register entries and comments; technical fields include action, tuning, wind and playing aids. Some keys are null. Register comments contain details that are not represented by the register name alone. The work year and last-change date are distinct attributes; reported measurements were not independently checked against the instrument." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The adapter assembles details, history, bibliography and media responses, with cached location context. Its detail model is distinct from the ordinary static-page family despite the common module.", "transformation": "Location, organ, builder and disposition objects are read separately. Technical mappings include pipe count, key and stop action, wind pressure, tuning, pitch, manual and pedal compass, accessories and notes. Stop count is calculated from the extracted division entries. Manual count prefers a stated field and falls back to a count derived from divisions. Events, bibliography and media use their own helpers.", "limitations": "Calculated stop totals are not independent source declarations; a division-derived manual count can differ from the physical console. History and principal builder information are merged conditionally. A scientific account must preserve the distinction between stated technical values, fallback context and calculated quantities." }, "implementation": { "class": "OrgelDocCrawler", "class_line": 2415, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrgelDocCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orgeldoc", "path": "aggregator/static_html_crawler.py", "line": 2416, "class": "OrgelDocCrawler" }, "id_column": { "value": "orgeldoc_id", "path": "aggregator/static_html_crawler.py", "line": 2417, "class": "OrgelDocCrawler" }, "default_country": { "value": "Switzerland", "path": "aggregator/static_html_crawler.py", "line": 2421, "class": "OrgelDocCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 30, "class": "StaticHtmlOrganCrawler" }, "detail_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 31, "class": "StaticHtmlOrganCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://orgeldokumentationszentrum.ch", "path": "aggregator/static_html_crawler.py", "line": 2419, "class": "OrgelDocCrawler" }, "api_base_url": { "value": "https://orgeldokumentationszentrum.ch/backend/v3", "path": "aggregator/static_html_crawler.py", "line": 2420, "class": "OrgelDocCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._building_name", "path": "aggregator/static_html_crawler.py", "line_start": 2705, "line_end": 2708, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 2708, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 2708, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 2706, "receiver": "standort", "key": "gebaeude", "expression": "standort.get('gebaeude')" }, { "line": 2706, "receiver": "standort", "key": "gebaeudeteil", "expression": "standort.get('gebaeudeteil')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._events", "path": "aggregator/static_html_crawler.py", "line_start": 2720, "line_end": 2736, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_meaningful_person" ], "literal_get_calls": [ { "line": 2722, "receiver": "history", "key": "orgelgeschichte", "expression": "history.get('orgelgeschichte', [])" }, { "line": 2722, "receiver": "history", "key": "orgelgeschichte", "expression": "history.get('orgelgeschichte')" }, { "line": 2730, "receiver": "row", "key": "person", "expression": "row.get('person')" }, { "line": 2731, "receiver": "row", "key": "aktion", "expression": "row.get('aktion')" }, { "line": 2728, "receiver": "row", "key": "person", "expression": "row.get('person', [])" }, { "line": 2732, "receiver": "row", "key": "datum", "expression": "row.get('datum')" }, { "line": 2732, "receiver": "row", "key": "datum", "expression": "row.get('datum')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2732, "key": "type", "value_expression": "activity" }, { "line": 2732, "key": "dates", "value_expression": "[self._clean_text(row.get('datum'))] if self._clean_text(row.get('datum')) else []" }, { "line": 2734, "key": "name", "value_expression": "person" }, { "line": 2734, "key": "activities", "value_expression": "[activity]" } ], "subscript_assignments": [ { "line": 2734, "value_line": 2734, "key": "involved", "target_expression": "event['involved']", "value_expression": "[{'name': person, 'activities': [activity]} for person in people]" } ] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._fetch_detail_bundle", "path": "aggregator/static_html_crawler.py", "line_start": 2501, "line_end": 2508, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_json", "_safe_fetch_json" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 2503, "key": "details", "value_expression": "self._fetch_json('details.php', params={'id': identifier})" }, { "line": 2504, "key": "history", "value_expression": "self._safe_fetch_json('details_orgelgeschichte.php', {'id': identifier})" }, { "line": 2505, "key": "bibliography", "value_expression": "self._safe_fetch_json('details_bibliographie.php', {'id': identifier})" }, { "line": 2506, "key": "media", "value_expression": "self._safe_fetch_json('details_medien.php', {'id': identifier})" }, { "line": 2504, "key": "id", "value_expression": "identifier" }, { "line": 2505, "key": "id", "value_expression": "identifier" }, { "line": 2506, "key": "id", "value_expression": "identifier" }, { "line": 2503, "key": "id", "value_expression": "identifier" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._fetch_json", "path": "aggregator/static_html_crawler.py", "line_start": 2450, "line_end": 2453, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 2451, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._literature", "path": "aggregator/static_html_crawler.py", "line_start": 2784, "line_end": 2794, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 2786, "receiver": "bibliography", "key": "bibliographie", "expression": "bibliography.get('bibliographie', [])" }, { "line": 2786, "receiver": "bibliography", "key": "bibliographie", "expression": "bibliography.get('bibliographie')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2793, "key": "raw_str", "value_expression": "text" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._media", "path": "aggregator/static_html_crawler.py", "line_start": 2775, "line_end": 2782, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 2777, "receiver": "media_bundle", "key": "medien", "expression": "media_bundle.get('medien', {})" }, { "line": 2778, "receiver": "medien", "key": "images", "expression": "medien.get('images', [])" }, { "line": 2777, "receiver": "media_bundle", "key": "medien", "expression": "media_bundle.get('medien')" }, { "line": 2778, "receiver": "medien", "key": "images", "expression": "medien.get('images')" }, { "line": 2779, "receiver": "image", "key": "src", "expression": "image.get('src')" }, { "line": 2781, "receiver": "image", "key": "src", "expression": "image.get('src')" }, { "line": 2781, "receiver": "image", "key": "beschreibung", "expression": "image.get('beschreibung')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2782, "key": "images", "value_expression": "images" }, { "line": 2781, "key": "url", "value_expression": "image.get('src')" }, { "line": 2781, "key": "caption", "value_expression": "self._clean_text(image.get('beschreibung'))" }, { "line": 2781, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 202, "line_end": 211, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 205, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._refs_from_geo_locations", "path": "aggregator/static_html_crawler.py", "line_start": 2455, "line_end": 2457, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_json", "_refs_from_geo_data" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._safe_fetch_json", "path": "aggregator/static_html_crawler.py", "line_start": 2510, "line_end": 2515, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_json" ], "literal_get_calls": [ { "line": 2514, "receiver": "params", "key": "id", "expression": "params.get('id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler._stops_by_division", "path": "aggregator/static_html_crawler.py", "line_start": 2748, "line_end": 2770, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 2751, "receiver": "disposition", "key": "dispositionen", "expression": "disposition.get('dispositionen', [])" }, { "line": 2751, "receiver": "disposition", "key": "dispositionen", "expression": "disposition.get('dispositionen')" }, { "line": 2756, "receiver": "division", "key": "tonhoehe", "expression": "division.get('tonhoehe', [])" }, { "line": 2754, "receiver": "division", "key": "werk", "expression": "division.get('werk')" }, { "line": 2756, "receiver": "division", "key": "tonhoehe", "expression": "division.get('tonhoehe')" }, { "line": 2759, "receiver": "stop", "key": "register", "expression": "stop.get('register')" }, { "line": 2763, "receiver": "stop", "key": "kommentar", "expression": "stop.get('kommentar')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2762, "key": "name", "value_expression": "stop_name" } ], "subscript_assignments": [ { "line": 2765, "value_line": 2765, "key": "remarks", "target_expression": "item['remarks']", "value_expression": "comment" } ] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 2423, "line_end": 2448, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_optional_int", "_refs_from_geo_locations", "_refs_from_search", "_save_reference", "_update_scrape_status" ], "literal_get_calls": [ { "line": 2426, "receiver": "kwargs", "key": "search", "expression": "kwargs.get('search')" }, { "line": 2424, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 2425, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 2608, "line_end": 2686, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_building_name", "_clean_text", "_coordinates", "_events", "_first_int", "_json_from_raw", "_literature", "_manual_count", "_meaningful_person", "_media", "_split_postal_town", "_stops_by_division" ], "literal_get_calls": [ { "line": 2610, "receiver": "bundle", "key": "details", "expression": "bundle.get('details')" }, { "line": 2611, "receiver": "bundle", "key": "history", "expression": "bundle.get('history')" }, { "line": 2612, "receiver": "bundle", "key": "bibliography", "expression": "bundle.get('bibliography')" }, { "line": 2613, "receiver": "bundle", "key": "media", "expression": "bundle.get('media')" }, { "line": 2616, "receiver": "details", "key": "standort", "expression": "details.get('standort', {})" }, { "line": 2617, "receiver": "details", "key": "orgel", "expression": "details.get('orgel', {})" }, { "line": 2618, "receiver": "details", "key": "orgelbauer", "expression": "details.get('orgelbauer', {})" }, { "line": 2619, "receiver": "details", "key": "disposition", "expression": "details.get('disposition', {})" }, { "line": 2638, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 2640, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 2609, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 2609, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 2610, "receiver": "bundle", "key": "details", "expression": "bundle.get('details')" }, { "line": 2611, "receiver": "bundle", "key": "history", "expression": "bundle.get('history')" }, { "line": 2612, "receiver": "bundle", "key": "bibliography", "expression": "bundle.get('bibliography')" }, { "line": 2613, "receiver": "bundle", "key": "media", "expression": "bundle.get('media')" }, { "line": 2614, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" }, { "line": 2616, "receiver": "details", "key": "standort", "expression": "details.get('standort')" }, { "line": 2617, "receiver": "details", "key": "orgel", "expression": "details.get('orgel')" }, { "line": 2618, "receiver": "details", "key": "orgelbauer", "expression": "details.get('orgelbauer')" }, { "line": 2619, "receiver": "details", "key": "disposition", "expression": "details.get('disposition')" }, { "line": 2621, "receiver": "standort", "key": "ort", "expression": "standort.get('ort')" }, { "line": 2621, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town', '')" }, { "line": 2627, "receiver": "standort", "key": "kanton", "expression": "standort.get('kanton')" }, { "line": 2627, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 2638, "receiver": "orgelbauer", "key": "personen_orgelbauer", "expression": "orgelbauer.get('personen_orgelbauer')" }, { "line": 2640, "receiver": "orgelbauer", "key": "personen_datum", "expression": "orgelbauer.get('personen_datum')" }, { "line": 2641, "receiver": "orgelbauer", "key": "personen_massnahme", "expression": "orgelbauer.get('personen_massnahme')" }, { "line": 2649, "receiver": "orgel", "key": "anzpfeifen", "expression": "orgel.get('anzpfeifen')" }, { "line": 2650, "receiver": "orgel", "key": "spieltraktur", "expression": "orgel.get('spieltraktur')" }, { "line": 2651, "receiver": "orgel", "key": "registertraktur", "expression": "orgel.get('registertraktur')" }, { "line": 2652, "receiver": "orgel", "key": "winddruck", "expression": "orgel.get('winddruck')" }, { "line": 2653, "receiver": "orgel", "key": "stimmung", "expression": "orgel.get('stimmung')" }, { "line": 2654, "receiver": "orgel", "key": "stimmtonhoehe", "expression": "orgel.get('stimmtonhoehe')" }, { "line": 2655, "receiver": "orgel", "key": "umfangmanuale", "expression": "orgel.get('umfangmanuale')" }, { "line": 2656, "receiver": "orgel", "key": "umfangpedale", "expression": "orgel.get('umfangpedale')" }, { "line": 2675, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 2676, "receiver": "details", "key": "letzteAenderung", "expression": "details.get('letzteAenderung', '')" }, { "line": 2683, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 2647, "receiver": "orgel", "key": "anzmanuale", "expression": "orgel.get('anzmanuale')" }, { "line": 2658, "receiver": "orgel", "key": "spielhilfen", "expression": "orgel.get('spielhilfen')" }, { "line": 2660, "receiver": "disposition", "key": "textdisposition", "expression": "disposition.get('textdisposition')" }, { "line": 2682, "receiver": "raw_data", "key": "orgeldoc_id", "expression": "raw_data.get('orgeldoc_id')" }, { "line": 2684, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 2657, "receiver": "orgel", "key": "spielhilfen", "expression": "orgel.get('spielhilfen', [])" }, { "line": 2663, "receiver": "orgel", "key": "kommentar", "expression": "orgel.get('kommentar')" }, { "line": 2663, "receiver": "orgel", "key": "bestandesaufnahme", "expression": "orgel.get('bestandesaufnahme')" }, { "line": 2673, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 2631, "receiver": "standort", "key": "ort", "expression": "standort.get('ort')" }, { "line": 2642, "receiver": "event.get('involved', [{}])[0]", "key": "name", "expression": "event.get('involved', [{}])[0].get('name')" }, { "line": 2642, "receiver": "event", "key": "involved", "expression": "event.get('involved')" }, { "line": 2642, "receiver": "event", "key": "involved", "expression": "event.get('involved', [{}])" } ], "html_queries": [], "dictionary_entries": [ { "line": 2625, "key": "status", "value_expression": "'current_location'" }, { "line": 2626, "key": "country", "value_expression": "self.default_country" }, { "line": 2627, "key": "region", "value_expression": "standort.get('kanton') or raw_data.get('region') or ''" }, { "line": 2628, "key": "town", "value_expression": "town" }, { "line": 2629, "key": "postcode", "value_expression": "postal_code" }, { "line": 2630, "key": "building", "value_expression": "building" }, { "line": 2631, "key": "raw_str", "value_expression": "', '.join((part for part in (standort.get('ort'), building) if part))" }, { "line": 2647, "key": "manual_count", "value_expression": "self._first_int(orgel.get('anzmanuale')) or self._manual_count(stops_by_division)" }, { "line": 2648, "key": "stop_count", "value_expression": "sum((len(stops) for stops in stops_by_division.values()))" }, { "line": 2649, "key": "pipe_count", "value_expression": "self._first_int(orgel.get('anzpfeifen'))" }, { "line": 2650, "key": "traction_key", "value_expression": "self._clean_text(orgel.get('spieltraktur'))" }, { "line": 2651, "key": "stop_action", "value_expression": "self._clean_text(orgel.get('registertraktur'))" }, { "line": 2652, "key": "wind_pressure", "value_expression": "self._clean_text(orgel.get('winddruck'))" }, { "line": 2653, "key": "tuning", "value_expression": "self._clean_text(orgel.get('stimmung'))" }, { "line": 2654, "key": "pitch", "value_expression": "self._clean_text(orgel.get('stimmtonhoehe'))" }, { "line": 2655, "key": "manual_compass", "value_expression": "self._clean_text(orgel.get('umfangmanuale'))" }, { "line": 2656, "key": "pedal_compass", "value_expression": "self._clean_text(orgel.get('umfangpedale'))" }, { "line": 2657, "key": "accessories", "value_expression": "[self._clean_text(item) for item in orgel.get('spielhilfen', []) if self._clean_text(item)] if isinstance(orgel.get('spielhilfen'), list) else []" }, { "line": 2660, "key": "raw_specification", "value_expression": "raw_spec or self._clean_text(disposition.get('textdisposition'))" }, { "line": 2661, "key": "stops", "value_expression": "stops_by_division" }, { "line": 2662, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 2663, "key": "notes", "value_expression": "[self._clean_text(value) for value in (orgel.get('kommentar'), orgel.get('bestandesaufnahme')) if self._clean_text(value)]" }, { "line": 2667, "key": "locations", "value_expression": "[location]" }, { "line": 2668, "key": "builder_history", "value_expression": "[{'builder_name': builder, 'year': year, 'activity': activity}] if builder else []" }, { "line": 2669, "key": "events", "value_expression": "events" }, { "line": 2670, "key": "specifications", "value_expression": "specifications" }, { "line": 2671, "key": "media", "value_expression": "self._media(media_bundle)" }, { "line": 2672, "key": "literature", "value_expression": "self._literature(bibliography)" }, { "line": 2673, "key": "links", "value_expression": "[{'link': raw_data.get('url') or f'{self.api_base_url}/details.php?id={raw_data.get(self.id_column)}'}]" }, { "line": 2674, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', ''), 'last_modified': details.get('letzteAenderung', '')}" }, { "line": 2675, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 2676, "key": "last_modified", "value_expression": "details.get('letzteAenderung', '')" }, { "line": 2643, "key": "type", "value_expression": "activity" }, { "line": 2643, "key": "dates", "value_expression": "[year] if year else []" }, { "line": 2643, "key": "involved", "value_expression": "[{'name': builder, 'activities': [activity]}]" }, { "line": 2673, "key": "link", "value_expression": "raw_data.get('url') or f'{self.api_base_url}/details.php?id={raw_data.get(self.id_column)}'" }, { "line": 2668, "key": "builder_name", "value_expression": "builder" }, { "line": 2668, "key": "year", "value_expression": "year" }, { "line": 2668, "key": "activity", "value_expression": "activity" }, { "line": 2643, "key": "name", "value_expression": "builder" }, { "line": 2643, "key": "activities", "value_expression": "[activity]" } ], "subscript_assignments": [ { "line": 2634, "value_line": 2634, "key": "latitude", "target_expression": "location['latitude']", "value_expression": "coordinates[0]" }, { "line": 2635, "value_line": 2635, "key": "longitude", "target_expression": "location['longitude']", "value_expression": "coordinates[1]" } ] } ], "selected_rule_rows": [ { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2648, "key": "stop_count", "value_expression": "sum((len(stops) for stops in stops_by_division.values()))" }, { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2647, "key": "manual_count", "value_expression": "self._first_int(orgel.get('anzmanuale')) or self._manual_count(stops_by_division)" }, { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2649, "key": "pipe_count", "value_expression": "self._first_int(orgel.get('anzpfeifen'))" }, { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2662, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2660, "key": "raw_specification", "value_expression": "raw_spec or self._clean_text(disposition.get('textdisposition'))" }, { "symbol": "OrgelDocCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2655, "key": "manual_compass", "value_expression": "self._clean_text(orgel.get('umfangmanuale'))" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelDocCrawler.transform", "line_start": 2645, "line_end": 2653, "text": " raw_spec, stops_by_division = self._stops_by_division(disposition)\n specifications = {\n \"manual_count\": self._first_int(orgel.get(\"anzmanuale\")) or self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"pipe_count\": self._first_int(orgel.get(\"anzpfeifen\")),\n \"traction_key\": self._clean_text(orgel.get(\"spieltraktur\")),\n \"stop_action\": self._clean_text(orgel.get(\"registertraktur\")),\n \"wind_pressure\": self._clean_text(orgel.get(\"winddruck\")),\n \"tuning\": self._clean_text(orgel.get(\"stimmung\"))," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelDocCrawler.transform", "line_start": 2644, "line_end": 2652, "text": "\n raw_spec, stops_by_division = self._stops_by_division(disposition)\n specifications = {\n \"manual_count\": self._first_int(orgel.get(\"anzmanuale\")) or self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"pipe_count\": self._first_int(orgel.get(\"anzpfeifen\")),\n \"traction_key\": self._clean_text(orgel.get(\"spieltraktur\")),\n \"stop_action\": self._clean_text(orgel.get(\"registertraktur\")),\n \"wind_pressure\": self._clean_text(orgel.get(\"winddruck\"))," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelDocCrawler.transform", "line_start": 2646, "line_end": 2654, "text": " specifications = {\n \"manual_count\": self._first_int(orgel.get(\"anzmanuale\")) or self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"pipe_count\": self._first_int(orgel.get(\"anzpfeifen\")),\n \"traction_key\": self._clean_text(orgel.get(\"spieltraktur\")),\n \"stop_action\": self._clean_text(orgel.get(\"registertraktur\")),\n \"wind_pressure\": self._clean_text(orgel.get(\"winddruck\")),\n \"tuning\": self._clean_text(orgel.get(\"stimmung\")),\n \"pitch\": self._clean_text(orgel.get(\"stimmtonhoehe\"))," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelDocCrawler.transform", "line_start": 2659, "line_end": 2667, "text": " else [],\n \"raw_specification\": raw_spec or self._clean_text(disposition.get(\"textdisposition\")),\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"notes\": [self._clean_text(value) for value in (orgel.get(\"kommentar\"), orgel.get(\"bestandesaufnahme\")) if self._clean_text(value)],\n }\n\n data = {\n \"locations\": [location]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelDocCrawler.transform", "line_start": 2657, "line_end": 2665, "text": " \"accessories\": [self._clean_text(item) for item in orgel.get(\"spielhilfen\", []) if self._clean_text(item)]\n if isinstance(orgel.get(\"spielhilfen\"), list)\n else [],\n \"raw_specification\": raw_spec or self._clean_text(disposition.get(\"textdisposition\")),\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"notes\": [self._clean_text(value) for value in (orgel.get(\"kommentar\"), orgel.get(\"bestandesaufnahme\")) if self._clean_text(value)],\n }\n" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelDocCrawler.transform", "line_start": 2652, "line_end": 2660, "text": " \"wind_pressure\": self._clean_text(orgel.get(\"winddruck\")),\n \"tuning\": self._clean_text(orgel.get(\"stimmung\")),\n \"pitch\": self._clean_text(orgel.get(\"stimmtonhoehe\")),\n \"manual_compass\": self._clean_text(orgel.get(\"umfangmanuale\")),\n \"pedal_compass\": self._clean_text(orgel.get(\"umfangpedale\")),\n \"accessories\": [self._clean_text(item) for item in orgel.get(\"spielhilfen\", []) if self._clean_text(item)]\n if isinstance(orgel.get(\"spielhilfen\"), list)\n else [],\n \"raw_specification\": raw_spec or self._clean_text(disposition.get(\"textdisposition\"))," } ] } }, { "catalogue_id": "S033", "source_key": "orgelsachsen", "name": "Orgeldatenbank Sachsen", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Hochschule für Musik Carl Maria von Weber Dresden", "scope_description": "Present-day Saxony, including historical instruments no longer extant", "documentary_unit": "Regional organ inventory record", "source_description": "The source describes a project founded in autumn 2015 with Protestant and Catholic church partners. It estimates approximately 3,500 known organs, including lost instruments, and about 2,500 surviving organs in Saxony.", "source_description_references": [ "orgelsachsen", "record_orgelsachsen" ], "interpretation": "These estimates describe the regional population, not a verified number of records exposed by the portal. Its territorial scope also differs from the church-owned instruments addressed by ORKASA, despite overlap between the two resources.", "rights_observation": "The university project displays copyright; no general open licence was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgelsachsen", "record_orgelsachsen" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgelsachsen", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Regional article with labelled organ description and historical specifications", "repeated_information": "Separate division tables for the later instrument and an earlier predecessor.", "date_and_state_distinctions": "The selected page contains an 1825 predecessor and a 1905 replacement, with a specification after a 1971 alteration.", "interpretive_limit": "Several technical labels are blank. A field heading is not a populated measurement.", "observations": [ "Street address and locality are given.", "A location-map link is supplied.", "Predecessor and successor builders are distinguished.", "Replacement and later redisposition are dated.", "Two historical configurations have separate stoplists.", "Manual/pedal divisions are explicit; exact key compasses are not given here.", "Couplers and ancillary controls are listed.", "The predecessor is explicitly described as no longer surviving.", "Bibliographic citation includes a page locator." ], "uninspected_or_unfilled_material": [ "Windchest and action labels exist, but no values appear in the selected record.", "Pitch label is present without a value.", "Gallery and image-credit headings exist without a verified image item." ], "narrative": "The Ablaß article contains separate specifications for a predecessor and a later instrument, including a later redisposition. Several action, windchest and pitch labels are unfilled. The specification headings therefore do not establish populated technical measurements. The predecessor is described as no longer surviving, which must not be transferred to the later instrument." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source page is acquired as HTML. Headline, location section, labelled technical data and tables supply different parts of the transfer.", "transformation": "The adapter separates a source-qualified identifier from Orgbase numbering. The headline is split for town/building, and location paragraphs can refine the address. Technical fields supply counts; action, windchest, pitch and repair information can remain descriptive notes. Tables with manual or pedal headers supply division-based stop rows. History is also retained as descriptive text.", "limitations": "A page may present historical specifications in addition to its principal disposition. Grouping table columns by division name does not itself provide a chronological configuration model. Notes retained as text remain valuable evidence even when no canonical field is populated. Default region and inferred title segments should be distinguished from explicit source labels." }, "implementation": { "class": "OrgSachsenCrawler", "class_line": 19, "path": "aggregator/orgelsachsen_crawler.py", "inheritance": [ "OrgSachsenCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "OrgSachsenCrawler.crawl", "path": "aggregator/orgelsachsen_crawler.py", "line_start": 40, "line_end": 165, "inherited": false, "file_sha256": "9926e666b10eda46e08d1e1a83e611e3d358fadad86410ee20785f7098541690", "self_calls": [ "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 44, "receiver": "kwargs", "key": "target", "expression": "kwargs.get('target')" }, { "line": 46, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 45, "receiver": "kwargs", "key": "page_limit", "expression": "kwargs.get('page_limit', 1000)" }, { "line": 102, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 110, "receiver": "params", "key": "articles", "expression": "params.get('articles', [None])" }, { "line": 98, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 98, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 96, "expression": "soup.select('div.mod_article a')" } ], "dictionary_entries": [ { "line": 117, "key": "orgelsachsen_id", "value_expression": "orgelsachsen_id" }, { "line": 118, "key": "url", "value_expression": "full_url" }, { "line": 119, "key": "status", "value_expression": "'initiated'" }, { "line": 120, "key": "scraped_at", "value_expression": "datetime.now()" } ], "subscript_assignments": [] }, { "symbol": "OrgSachsenCrawler.transform", "path": "aggregator/orgelsachsen_crawler.py", "line_start": 295, "line_end": 605, "inherited": false, "file_sha256": "9926e666b10eda46e08d1e1a83e611e3d358fadad86410ee20785f7098541690", "self_calls": [], "literal_get_calls": [ { "line": 299, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 300, "receiver": "raw_data", "key": "orgelsachsen_id", "expression": "raw_data.get('orgelsachsen_id')" }, { "line": 301, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 422, "receiver": "tech_data", "key": "erbauer", "expression": "tech_data.get('erbauer')" }, { "line": 423, "receiver": "tech_data", "key": "baujahr", "expression": "tech_data.get('baujahr')" }, { "line": 546, "receiver": "tech_data", "key": "geschichte", "expression": "tech_data.get('geschichte')" }, { "line": 570, "receiver": "media_images[0]", "key": "link", "expression": "media_images[0].get('link')" }, { "line": 590, "receiver": "a", "key": "href", "expression": "a.get('href')" }, { "line": 579, "receiver": "link_tag", "key": "href", "expression": "link_tag.get('href')" }, { "line": 579, "receiver": "img_tag", "key": "src", "expression": "img_tag.get('src')" } ], "html_queries": [ { "line": 334, "expression": "soup.find('h2', class_='ce_headline')" }, { "line": 361, "expression": "soup.find('h2', string=re.compile('Standort', re.I))" }, { "line": 416, "expression": "soup.find_all('table')" }, { "line": 468, "expression": "soup.find_all('table')" }, { "line": 515, "expression": "soup.find_all(['h2', 'h3'])" }, { "line": 529, "expression": "soup.find(['h2', 'h3'], string=re.compile('Bibliographie|Literatur', re.I))" }, { "line": 549, "expression": "soup.find(['h2', 'h3'], string=re.compile('Geschichte', re.I))" }, { "line": 574, "expression": "soup.find('figure', class_='image_container')" }, { "line": 587, "expression": "soup.find('div', class_='ce_gallery')" }, { "line": 328, "expression": "soup.find(string=pattern)" }, { "line": 379, "expression": "table.find_all('tr')" }, { "line": 418, "expression": "table.find(string=re.compile('Erbauer|Baujahr', re.I))" }, { "line": 470, "expression": "table.find(string=re.compile('Erbauer|Baujahr', re.I))" }, { "line": 473, "expression": "table.find('thead')" }, { "line": 576, "expression": "figure.find('img')" }, { "line": 577, "expression": "figure.find('a')" }, { "line": 589, "expression": "gallery.find_all('a')" }, { "line": 381, "expression": "tr.find('th')" }, { "line": 382, "expression": "tr.find('td')" }, { "line": 388, "expression": "tr.find_all('td')" }, { "line": 477, "expression": "table.find('tbody')" }, { "line": 519, "expression": "table.find_all('td')" }, { "line": 475, "expression": "thead.find_all('th')" }, { "line": 480, "expression": "tbody.find_all('tr')" }, { "line": 481, "expression": "tr.find_all('td')" } ], "dictionary_entries": [ { "line": 314, "key": "name", "value_expression": "'Orgeldatenbank Sachsen Identifier'" }, { "line": 315, "key": "id", "value_expression": "orgelsachsen_id" }, { "line": 316, "key": "host", "value_expression": "'orgelforum-sachsen.de'" }, { "line": 317, "key": "hostcode", "value_expression": "'orgelsachsen'" }, { "line": 320, "key": "type", "value_expression": "'source'" }, { "line": 321, "key": "host", "value_expression": "'orgelforum-sachsen.de'" }, { "line": 322, "key": "link", "value_expression": "url" }, { "line": 337, "key": "status", "value_expression": "'current_location'" }, { "line": 338, "key": "country", "value_expression": "'Germany'" }, { "line": 339, "key": "region", "value_expression": "'Sachsen'" }, { "line": 340, "key": "town", "value_expression": "''" }, { "line": 341, "key": "building", "value_expression": "''" }, { "line": 342, "key": "address", "value_expression": "''" }, { "line": 343, "key": "raw_str", "value_expression": "''" }, { "line": 426, "key": "name", "value_expression": "builder" }, { "line": 427, "key": "activities", "value_expression": "['builder']" }, { "line": 428, "key": "dates", "value_expression": "[year] if year else []" }, { "line": 432, "key": "type", "value_expression": "'construction'" }, { "line": 433, "key": "date", "value_expression": "year or ''" }, { "line": 434, "key": "description", "value_expression": "builder or 'Source-reported construction'" }, { "line": 435, "key": "raw_str", "value_expression": "' · '.join((value for value in (builder, year) if value))" }, { "line": 581, "key": "link", "value_expression": "urljoin(self.base_url, img_src)" }, { "line": 582, "key": "filename", "value_expression": "os.path.basename(img_src)" }, { "line": 583, "key": "available", "value_expression": "True" }, { "line": 593, "key": "link", "value_expression": "urljoin(self.base_url, img_src)" }, { "line": 594, "key": "filename", "value_expression": "os.path.basename(img_src)" }, { "line": 595, "key": "available", "value_expression": "True" }, { "line": 538, "key": "type", "value_expression": "'unclassified'" }, { "line": 539, "key": "title", "value_expression": "text" }, { "line": 540, "key": "raw_str", "value_expression": "text" }, { "line": 524, "key": "raw_str", "value_expression": "item" }, { "line": 526, "key": "raw_str", "value_expression": "item" }, { "line": 502, "key": "name", "value_expression": "stop_match.group(1).strip()" }, { "line": 503, "key": "length", "value_expression": "stop_match.group(2).strip()" }, { "line": 504, "key": "raw_str", "value_expression": "line" }, { "line": 508, "key": "name", "value_expression": "clean_line" }, { "line": 509, "key": "raw_str", "value_expression": "line" } ], "subscript_assignments": [ { "line": 312, "value_line": 312, "key": "orgbase_id", "target_expression": "structured_entry['orgbase_id']", "value_expression": "None" }, { "line": 358, "value_line": 358, "key": "country", "target_expression": "location['country']", "value_expression": "'Germany'" }, { "line": 359, "value_line": 359, "key": "region", "target_expression": "location['region']", "value_expression": "'Sachsen'" }, { "line": 441, "value_line": 441, "key": "manuals", "target_expression": "specs['manuals']", "value_expression": "[]" }, { "line": 442, "value_line": 442, "key": "pedals", "target_expression": "specs['pedals']", "value_expression": "[]" }, { "line": 443, "value_line": 443, "key": "couplers", "target_expression": "specs['couplers']", "value_expression": "[]" }, { "line": 444, "value_line": 444, "key": "accessories", "target_expression": "specs['accessories']", "value_expression": "[]" }, { "line": 445, "value_line": 445, "key": "werke", "target_expression": "specs['werke']", "value_expression": "[]" }, { "line": 448, "value_line": 448, "key": "manual_count", "target_expression": "specs['manual_count']", "value_expression": "tech_data['manuale']" }, { "line": 450, "value_line": 450, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "tech_data['register']" }, { "line": 512, "value_line": 512, "key": "stops", "target_expression": "specs['stops']", "value_expression": "[all_werk_stops]" }, { "line": 354, "value_line": 354, "key": "town", "target_expression": "location['town']", "value_expression": "parts[0]" }, { "line": 356, "value_line": 356, "key": "building", "target_expression": "location['building']", "value_expression": "parts[1]" }, { "line": 367, "value_line": 367, "key": "address", "target_expression": "location['address']", "value_expression": "addr_lines[0]" }, { "line": 370, "value_line": 370, "key": "raw_str", "target_expression": "location['raw_str']", "value_expression": "' '.join(addr_lines)" }, { "line": 374, "value_line": 374, "key": "town", "target_expression": "location['town']", "value_expression": "town_match.group(2)" } ] } ], "selected_rule_rows": [ { "symbol": "OrgSachsenCrawler.transform", "path": "aggregator/orgelsachsen_crawler.py", "line": 450, "value_line": 450, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "tech_data['register']" }, { "symbol": "OrgSachsenCrawler.transform", "path": "aggregator/orgelsachsen_crawler.py", "line": 448, "value_line": 448, "key": "manual_count", "target_expression": "specs['manual_count']", "value_expression": "tech_data['manuale']" }, { "symbol": "OrgSachsenCrawler.transform", "path": "aggregator/orgelsachsen_crawler.py", "line": 428, "key": "dates", "value_expression": "[year] if year else []" }, { "symbol": "OrgSachsenCrawler.transform", "path": "aggregator/orgelsachsen_crawler.py", "line": 338, "key": "country", "value_expression": "'Germany'" }, { "symbol": "OrgSachsenCrawler.transform", "path": "aggregator/orgelsachsen_crawler.py", "line": 343, "key": "raw_str", "value_expression": "''" } ], "bounded_code_passages": [ { "path": "aggregator/orgelsachsen_crawler.py", "symbol": "OrgSachsenCrawler.transform", "line_start": 447, "line_end": 455, "text": " if \"manuale\" in tech_data:\n specs[\"manual_count\"] = tech_data[\"manuale\"]\n if \"register\" in tech_data:\n specs[\"stop_count\"] = tech_data[\"register\"]\n \n # Add technical details to descriptions\n # We also look for 'geschichte' here now, as it might be captured by the new table parser\n tech_notes = []\n for key in [\"windladen\", \"spieltraktur\", \"registratur\", \"stimmtonh\\u00f6he\", \"balganlage\", \"orgeldetails\", \"umbauten / reparaturen\"]:" }, { "path": "aggregator/orgelsachsen_crawler.py", "symbol": "OrgSachsenCrawler.transform", "line_start": 445, "line_end": 453, "text": " specs[\"werke\"] = []\n\n if \"manuale\" in tech_data:\n specs[\"manual_count\"] = tech_data[\"manuale\"]\n if \"register\" in tech_data:\n specs[\"stop_count\"] = tech_data[\"register\"]\n \n # Add technical details to descriptions\n # We also look for 'geschichte' here now, as it might be captured by the new table parser" }, { "path": "aggregator/orgelsachsen_crawler.py", "symbol": "OrgSachsenCrawler.transform", "line_start": 425, "line_end": 433, "text": " structured_entry[\"involved\"].append({\n \"name\": builder,\n \"activities\": [\"builder\"],\n \"dates\": [year] if year else []\n })\n if builder or year:\n structured_entry[\"events\"].append({\n \"type\": \"construction\",\n \"date\": year or \"\"," }, { "path": "aggregator/orgelsachsen_crawler.py", "symbol": "OrgSachsenCrawler.transform", "line_start": 335, "line_end": 343, "text": " if not structured_entry[\"locations\"]:\n structured_entry[\"locations\"].append({\n \"status\": \"current_location\",\n \"country\": \"Germany\",\n \"region\": \"Sachsen\",\n \"town\": \"\",\n \"building\": \"\",\n \"address\": \"\",\n \"raw_str\": \"\"" }, { "path": "aggregator/orgelsachsen_crawler.py", "symbol": "OrgSachsenCrawler.transform", "line_start": 340, "line_end": 348, "text": " \"town\": \"\",\n \"building\": \"\",\n \"address\": \"\",\n \"raw_str\": \"\"\n })\n \n location = structured_entry[\"locations\"][0]\n \n if headline:" } ] } }, { "catalogue_id": "S034", "source_key": "orgelseite", "name": "Die Orgelseite", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Die Orgelseite", "scope_description": "International", "documentary_unit": "Organ profile with photographs and specification", "source_description": "The home page reported 6,622 organs in 83 countries and an update date of 5 September 2026. The imprint distinguishes public access from subscription and music-download areas and acknowledges photographs supplied by organ builders and other contributors.", "source_description_references": [ "orgelseite", "orgelseite_imprint", "record_orgelseite_list" ], "interpretation": "These totals are publisher statements. They do not measure data completeness or unrestricted access. The operator's name was not present in the retrieved imprint text, so the citation uses the publication title.", "rights_observation": "Text and photographs are described as copyrighted; contributed photographs may retain third-party rights.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgelseite", "orgelseite_imprint", "record_orgelseite_list" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgelseite_list", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Filterable multi-record HTML table with links to specifications, photographs and music", "repeated_information": "One table row per listed organ, with optional columns for ranks, stops, pipes and documentary sources.", "date_and_state_distinctions": "Build-year ranges, information stand and last-update fields are distinct. Historical stoplists are counted separately.", "interpretive_limit": "Displayed row number is an ordering position, not an established persistent organ identifier. Extensions, transmissions and digital additions are separate columns.", "observations": [ "Country, region, city and venue are populated in the organ table.", "Rows link to organ locations on maps.", "Builder names and year ranges are table columns.", "Build-year ranges represent documentary chronology, without an explicit event model in this table.", "Manual counts are tabulated.", "Rows identify publications, direct contacts and other websites.", "Rows link to specification, gallery and external source pages.", "Information stand and last-update are offered as separate optional columns." ], "uninspected_or_unfilled_material": [ "Specification links are present; linked stoplist files were not opened.", "Image and gallery links are present; their payloads were not inspected.", "CD and MP3 columns link to the separate music catalogue." ], "narrative": "The inspected list presents organs in rows with filters and optional columns. Stops, ranks, extensions, transmissions and digital additions are distinguished, and rows link to specifications, images, maps and music. Documentary sources and update-related fields are also exposed. Linked media and specification files were not opened in this check. A displayed row position was not established as a persistent organ identifier." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "A catalogue row, its disposition page and any acquired disposition workbook are distinct inputs. The source number is qualified as an Orgelseite identifier. The transform retains the catalogue cells and records which detail representation supplies the specification.", "transformation": "Location and initial builder attribution come from named catalogue columns. Both HTML and spreadsheet specification parsers are called; the spreadsheet result is selected when its parser reports a complete stoplist, otherwise the HTML result is selected. Divisions, stops, counts and raw specification text follow that choice. Photograph, gallery, audio and map links remain distinct. Non-empty catalogue cells are retained in a source-specific extension.", "limitations": "The completeStopList flag is the parser's criterion, not independent confirmation of exhaustive organ documentation. A map link does not become a measured coordinate. A listing builder/year pair is projected as construction, so later interventions or abbreviated listings need documentary review. Workbook and HTML states should not be presumed contemporaneous." }, "implementation": { "class": "OrgelseiteCrawler", "class_line": 352, "path": "aggregator/orgelseite_crawler.py", "inheritance": [ "OrgelseiteCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "OrgelseiteCrawler.crawl", "path": "aggregator/orgelseite_crawler.py", "line_start": 527, "line_end": 657, "inherited": false, "file_sha256": "a0a6abf20fcdf54a32e45957cdaa93e373df7016b435d9f8093efe06f8b4e716", "self_calls": [ "_find_organ_list_table", "_is_natural_list_exhaustion" ], "literal_get_calls": [ { "line": 532, "receiver": "kwargs", "key": "target", "expression": "kwargs.get('target', 'manhattan')" }, { "line": 564, "receiver": "kwargs", "key": "max_pages", "expression": "kwargs.get('max_pages', kwargs.get('page_count', 1000))" }, { "line": 564, "receiver": "kwargs", "key": "page_count", "expression": "kwargs.get('page_count', 1000)" }, { "line": 614, "receiver": "raw_entry", "key": "Nr", "expression": "raw_entry.get('Nr')" }, { "line": 615, "receiver": "raw_entry", "key": "Disposition", "expression": "raw_entry.get('Disposition')" }, { "line": 617, "receiver": "raw_entry", "key": "Bundesland / Region", "expression": "raw_entry.get('Bundesland / Region')" }, { "line": 618, "receiver": "raw_entry", "key": "Stadt", "expression": "raw_entry.get('Stadt')" }, { "line": 619, "receiver": "raw_entry", "key": "Kirche / Saal", "expression": "raw_entry.get('Kirche / Saal')" }, { "line": 620, "receiver": "raw_entry", "key": "Erbauer", "expression": "raw_entry.get('Erbauer')" }, { "line": 621, "receiver": "raw_entry", "key": "Baujahr", "expression": "raw_entry.get('Baujahr')" }, { "line": 622, "receiver": "raw_entry", "key": "Manuale", "expression": "raw_entry.get('Manuale')" }, { "line": 623, "receiver": "raw_entry", "key": "Register", "expression": "raw_entry.get('Register')" }, { "line": 624, "receiver": "raw_entry", "key": "Pfeifen", "expression": "raw_entry.get('Pfeifen')" }, { "line": 602, "receiver": "link_tag", "key": "href", "expression": "link_tag.get('href')" }, { "line": 603, "receiver": "link_tag", "key": "href", "expression": "link_tag.get('href')" }, { "line": 616, "receiver": "raw_entry", "key": "Land", "expression": "raw_entry.get('Land', '')" }, { "line": 616, "receiver": "raw_entry", "key": "Land", "expression": "raw_entry.get('Land', '')" } ], "html_queries": [ { "line": 594, "expression": "table.find_all('tr')" }, { "line": 595, "expression": "tr.find_all('td')" }, { "line": 601, "expression": "value_td.find('a')" }, { "line": 608, "expression": "value_td.find_all('br')" } ], "dictionary_entries": [ { "line": 614, "key": "orgelseite_id", "value_expression": "raw_entry.get('Nr')" }, { "line": 615, "key": "url", "value_expression": "raw_entry.get('Disposition')" }, { "line": 616, "key": "country", "value_expression": "self.country_mapping.get(raw_entry.get('Land', ''), raw_entry.get('Land', ''))" }, { "line": 617, "key": "region", "value_expression": "raw_entry.get('Bundesland / Region')" }, { "line": 618, "key": "town", "value_expression": "raw_entry.get('Stadt')" }, { "line": 619, "key": "building", "value_expression": "raw_entry.get('Kirche / Saal')" }, { "line": 620, "key": "builder", "value_expression": "raw_entry.get('Erbauer')" }, { "line": 621, "key": "year", "value_expression": "raw_entry.get('Baujahr')" }, { "line": 622, "key": "manuals", "value_expression": "raw_entry.get('Manuale')" }, { "line": 623, "key": "stops", "value_expression": "raw_entry.get('Register')" }, { "line": 624, "key": "pipes", "value_expression": "raw_entry.get('Pfeifen')" }, { "line": 625, "key": "raw_data_json", "value_expression": "json.dumps(raw_entry)" }, { "line": 626, "key": "scraped_at", "value_expression": "datetime.now()" } ], "subscript_assignments": [] }, { "symbol": "OrgelseiteCrawler.crawl_images", "path": "aggregator/orgelseite_crawler.py", "line_start": 878, "line_end": 974, "inherited": false, "file_sha256": "a0a6abf20fcdf54a32e45957cdaa93e373df7016b435d9f8093efe06f8b4e716", "self_calls": [], "literal_get_calls": [ { "line": 943, "receiver": "raw_image_data", "key": "Foto", "expression": "raw_image_data.get('Foto')" }, { "line": 948, "receiver": "raw_image_data", "key": "Galerie", "expression": "raw_image_data.get('Galerie', '')" }, { "line": 957, "receiver": "raw_image_data", "key": "Bildmotiv", "expression": "raw_image_data.get('Bildmotiv')" }, { "line": 937, "receiver": "link_tag", "key": "href", "expression": "link_tag.get('href')" }, { "line": 938, "receiver": "link_tag", "key": "href", "expression": "link_tag.get('href')" } ], "html_queries": [ { "line": 924, "expression": "soup.find('table', {'border': '1', 'cellpadding': '2', 'cellspacing': '0'})" }, { "line": 929, "expression": "table.find_all('tr')" }, { "line": 930, "expression": "tr.find_all('td')" }, { "line": 936, "expression": "value_td.find('a')" } ], "dictionary_entries": [ { "line": 924, "key": "border", "value_expression": "'1'" }, { "line": 924, "key": "cellpadding", "value_expression": "'2'" }, { "line": 924, "key": "cellspacing", "value_expression": "'0'" }, { "line": 954, "key": "orgelseite_id", "value_expression": "orgelseite_id" }, { "line": 955, "key": "image_url", "value_expression": "image_url" }, { "line": 956, "key": "gallery_url", "value_expression": "gallery_url" }, { "line": 957, "key": "description", "value_expression": "raw_image_data.get('Bildmotiv')" }, { "line": 958, "key": "raw_image_data_json", "value_expression": "json.dumps(raw_image_data)" }, { "line": 959, "key": "scraped_at", "value_expression": "datetime.now()" } ], "subscript_assignments": [] }, { "symbol": "OrgelseiteCrawler.transform", "path": "aggregator/orgelseite_crawler.py", "line_start": 659, "line_end": 798, "inherited": false, "file_sha256": "a0a6abf20fcdf54a32e45957cdaa93e373df7016b435d9f8093efe06f8b4e716", "self_calls": [], "literal_get_calls": [ { "line": 707, "receiver": "raw_data", "key": "Google Maps", "expression": "raw_data.get('Google Maps')" }, { "line": 721, "receiver": "raw_data", "key": "Bundesland / Region", "expression": "raw_data.get('Bundesland / Region')" }, { "line": 722, "receiver": "raw_data", "key": "Stadt", "expression": "raw_data.get('Stadt')" }, { "line": 723, "receiver": "raw_data", "key": "Kirche / Saal", "expression": "raw_data.get('Kirche / Saal')" }, { "line": 724, "receiver": "raw_data", "key": "Bezirk", "expression": "raw_data.get('Bezirk')" }, { "line": 727, "receiver": "raw_data", "key": "Erbauer", "expression": "raw_data.get('Erbauer')" }, { "line": 783, "receiver": "raw_data", "key": "Foto", "expression": "raw_data.get('Foto')" }, { "line": 785, "receiver": "raw_data", "key": "Fotogalerie", "expression": "raw_data.get('Fotogalerie')" }, { "line": 787, "receiver": "raw_data", "key": "MP3", "expression": "raw_data.get('MP3')" }, { "line": 720, "receiver": "raw_data", "key": "Land", "expression": "raw_data.get('Land', '')" }, { "line": 720, "receiver": "raw_data", "key": "Land", "expression": "raw_data.get('Land', '')" }, { "line": 755, "receiver": "raw_data", "key": "Info-Quellen / Links", "expression": "raw_data.get('Info-Quellen / Links', '')" }, { "line": 683, "receiver": "raw_data", "key": "Nr", "expression": "raw_data.get('Nr')" }, { "line": 692, "receiver": "raw_data", "key": "Disposition", "expression": "raw_data.get('Disposition', '')" }, { "line": 740, "receiver": "raw_data", "key": "Manuale", "expression": "raw_data.get('Manuale')" }, { "line": 741, "receiver": "raw_data", "key": "Register", "expression": "raw_data.get('Register')" }, { "line": 794, "receiver": "raw_data", "key": "Disposition", "expression": "raw_data.get('Disposition', '')" }, { "line": 697, "receiver": "raw_data", "key": "Disposition", "expression": "raw_data.get('Disposition')" }, { "line": 712, "receiver": "raw_data", "key": "Google Maps", "expression": "raw_data.get('Google Maps')" }, { "line": 784, "receiver": "raw_data", "key": "Foto", "expression": "raw_data.get('Foto')" }, { "line": 786, "receiver": "raw_data", "key": "Fotogalerie", "expression": "raw_data.get('Fotogalerie')" }, { "line": 788, "receiver": "raw_data", "key": "MP3", "expression": "raw_data.get('MP3')" }, { "line": 793, "receiver": "raw_data", "key": "Nr", "expression": "raw_data.get('Nr')" }, { "line": 793, "receiver": "raw_data", "key": "orgelseite_id", "expression": "raw_data.get('orgelseite_id')" }, { "line": 732, "receiver": "raw_data", "key": "Baujahr", "expression": "raw_data.get('Baujahr', '')" }, { "line": 731, "receiver": "raw_data", "key": "Erbauer", "expression": "raw_data.get('Erbauer')" } ], "html_queries": [], "dictionary_entries": [ { "line": 762, "key": "source_field_language", "value_expression": "'de'" }, { "line": 763, "key": "listing", "value_expression": "{str(key): value for key, value in raw_data.items() if value not in (None, '')}" }, { "line": 768, "key": "detail_specification_parser", "value_expression": "detail_specification['parser']" }, { "line": 769, "key": "detail_specification_sources", "value_expression": "{'selected': 'excel' if detail_specification is excel_specification else 'html', 'html': html_specification['parser'], 'excel': excel_specification['parser'], 'excel_url': excel_url}" }, { "line": 682, "key": "name", "value_expression": "'Die Orgelseite Number'" }, { "line": 683, "key": "id", "value_expression": "raw_data.get('Nr')" }, { "line": 684, "key": "host", "value_expression": "'die-orgelseite.de'" }, { "line": 685, "key": "hostcode", "value_expression": "'orgelseite'" }, { "line": 690, "key": "type", "value_expression": "'source'" }, { "line": 691, "key": "host", "value_expression": "'die-orgelseite.de'" }, { "line": 692, "key": "link", "value_expression": "raw_data.get('Disposition', '')" }, { "line": 771, "key": "selected", "value_expression": "'excel' if detail_specification is excel_specification else 'html'" }, { "line": 775, "key": "html", "value_expression": "html_specification['parser']" }, { "line": 776, "key": "excel", "value_expression": "excel_specification['parser']" }, { "line": 777, "key": "excel_url", "value_expression": "excel_url" }, { "line": 702, "key": "type", "value_expression": "'specification-data'" }, { "line": 703, "key": "host", "value_expression": "'die-orgelseite.de'" }, { "line": 704, "key": "link", "value_expression": "excel_url" }, { "line": 710, "key": "type", "value_expression": "'map'" }, { "line": 711, "key": "host", "value_expression": "'google.com'" }, { "line": 712, "key": "link", "value_expression": "raw_data.get('Google Maps')" }, { "line": 730, "key": "type", "value_expression": "'construction'" }, { "line": 731, "key": "involved", "value_expression": "[{'name': raw_data.get('Erbauer'), 'activities': ['builder']}]" }, { "line": 732, "key": "dates", "value_expression": "[raw_data.get('Baujahr', '')]" }, { "line": 784, "key": "link", "value_expression": "raw_data.get('Foto')" }, { "line": 784, "key": "source", "value_expression": "'Die Orgelseite'" }, { "line": 786, "key": "link", "value_expression": "raw_data.get('Fotogalerie')" }, { "line": 786, "key": "source", "value_expression": "'Die Orgelseite'" }, { "line": 788, "key": "link", "value_expression": "raw_data.get('MP3')" }, { "line": 788, "key": "source", "value_expression": "'Die Orgelseite'" }, { "line": 731, "key": "name", "value_expression": "raw_data.get('Erbauer')" }, { "line": 731, "key": "activities", "value_expression": "['builder']" } ], "subscript_assignments": [ { "line": 720, "value_line": 720, "key": "country", "target_expression": "location['country']", "value_expression": "self.country_mapping.get(raw_data.get('Land', ''), raw_data.get('Land', ''))" }, { "line": 721, "value_line": 721, "key": "region", "target_expression": "location['region']", "value_expression": "raw_data.get('Bundesland / Region')" }, { "line": 722, "value_line": 722, "key": "town", "target_expression": "location['town']", "value_expression": "raw_data.get('Stadt')" }, { "line": 723, "value_line": 723, "key": "building", "target_expression": "location['building']", "value_expression": "raw_data.get('Kirche / Saal')" }, { "line": 724, "value_line": 724, "key": "address", "target_expression": "location['address']", "value_expression": "raw_data.get('Bezirk')" }, { "line": 749, "value_line": 749, "key": "manual_count", "target_expression": "specs['manual_count']", "value_expression": "detail_specification['manual_count']" }, { "line": 750, "value_line": 750, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "detail_specification['stop_count']" }, { "line": 751, "value_line": 751, "key": "divisions", "target_expression": "specs['divisions']", "value_expression": "detail_specification['divisions']" }, { "line": 752, "value_line": 752, "key": "stops", "target_expression": "specs['stops']", "value_expression": "detail_specification['stops']" }, { "line": 753, "value_line": 753, "key": "stops_by_division", "target_expression": "specs['stops_by_division']", "value_expression": "detail_specification['stops_by_division']" }, { "line": 754, "value_line": 754, "key": "raw_spec", "target_expression": "specs['raw_spec']", "value_expression": "detail_specification['raw_spec']" }, { "line": 755, "value_line": 755, "key": "specification_notes", "target_expression": "specs['specification_notes']", "value_expression": "[raw_data.get('Info-Quellen / Links', '')]" }, { "line": 761, "value_line": 761, "key": "orgelseite", "target_expression": "structured_entry.setdefault('extensions', {})['orgelseite']", "value_expression": "{'source_field_language': 'de', 'listing': {str(key): value for key, value in raw_data.items() if value not in (None, '')}, 'detail_specification_parser': detail_specification['parser'], 'detail_specification_sources': {'selected': 'excel' if detail_specification is excel_specification else 'html', 'html': html_specification['parser'], 'excel': excel_specification['parser'], 'excel_url': excel_url}}" } ] } ], "selected_rule_rows": [ { "symbol": "OrgelseiteCrawler.transform", "path": "aggregator/orgelseite_crawler.py", "line": 750, "value_line": 750, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "detail_specification['stop_count']" }, { "symbol": "OrgelseiteCrawler.transform", "path": "aggregator/orgelseite_crawler.py", "line": 749, "value_line": 749, "key": "manual_count", "target_expression": "specs['manual_count']", "value_expression": "detail_specification['manual_count']" }, { "symbol": "OrgelseiteCrawler.transform", "path": "aggregator/orgelseite_crawler.py", "line": 753, "value_line": 753, "key": "stops_by_division", "target_expression": "specs['stops_by_division']", "value_expression": "detail_specification['stops_by_division']" }, { "symbol": "OrgelseiteCrawler.transform", "path": "aggregator/orgelseite_crawler.py", "line": 732, "key": "dates", "value_expression": "[raw_data.get('Baujahr', '')]" }, { "symbol": "OrgelseiteCrawler.transform", "path": "aggregator/orgelseite_crawler.py", "line": 720, "value_line": 720, "key": "country", "target_expression": "location['country']", "value_expression": "self.country_mapping.get(raw_data.get('Land', ''), raw_data.get('Land', ''))" } ], "bounded_code_passages": [ { "path": "aggregator/orgelseite_crawler.py", "symbol": "OrgelseiteCrawler.transform", "line_start": 747, "line_end": 755, "text": " else html_specification\n )\n specs[\"manual_count\"] = detail_specification[\"manual_count\"]\n specs[\"stop_count\"] = detail_specification[\"stop_count\"]\n specs[\"divisions\"] = detail_specification[\"divisions\"]\n specs[\"stops\"] = detail_specification[\"stops\"]\n specs[\"stops_by_division\"] = detail_specification[\"stops_by_division\"]\n specs[\"raw_spec\"] = detail_specification[\"raw_spec\"]\n specs[\"specification_notes\"] = [raw_data.get(\"Info-Quellen / Links\", \"\")]" }, { "path": "aggregator/orgelseite_crawler.py", "symbol": "OrgelseiteCrawler.transform", "line_start": 746, "line_end": 754, "text": " if excel_specification[\"parser\"][\"completeStopList\"]\n else html_specification\n )\n specs[\"manual_count\"] = detail_specification[\"manual_count\"]\n specs[\"stop_count\"] = detail_specification[\"stop_count\"]\n specs[\"divisions\"] = detail_specification[\"divisions\"]\n specs[\"stops\"] = detail_specification[\"stops\"]\n specs[\"stops_by_division\"] = detail_specification[\"stops_by_division\"]\n specs[\"raw_spec\"] = detail_specification[\"raw_spec\"]" }, { "path": "aggregator/orgelseite_crawler.py", "symbol": "OrgelseiteCrawler.transform", "line_start": 750, "line_end": 758, "text": " specs[\"stop_count\"] = detail_specification[\"stop_count\"]\n specs[\"divisions\"] = detail_specification[\"divisions\"]\n specs[\"stops\"] = detail_specification[\"stops\"]\n specs[\"stops_by_division\"] = detail_specification[\"stops_by_division\"]\n specs[\"raw_spec\"] = detail_specification[\"raw_spec\"]\n specs[\"specification_notes\"] = [raw_data.get(\"Info-Quellen / Links\", \"\")]\n\n # Retain every non-empty catalogue cell under a source-specific,\n # language-qualified extension. Canonical mappings above remain" }, { "path": "aggregator/orgelseite_crawler.py", "symbol": "OrgelseiteCrawler.transform", "line_start": 729, "line_end": 737, "text": " {\n \"type\": \"construction\",\n \"involved\": [{\"name\": raw_data.get(\"Erbauer\"), \"activities\": [\"builder\"]}],\n \"dates\": [raw_data.get(\"Baujahr\", \"\")],\n }\n )\n\n # Specifications\n specs = structured_entry[\"specifications\"]" }, { "path": "aggregator/orgelseite_crawler.py", "symbol": "OrgelseiteCrawler.transform", "line_start": 717, "line_end": 725, "text": " if not structured_entry[\"locations\"]:\n structured_entry[\"locations\"].append({})\n location = structured_entry[\"locations\"][0]\n location[\"country\"] = self.country_mapping.get(raw_data.get(\"Land\", \"\"), raw_data.get(\"Land\", \"\"))\n location[\"region\"] = raw_data.get(\"Bundesland / Region\") # Corrected from raw_entry\n location[\"town\"] = raw_data.get(\"Stadt\")\n location[\"building\"] = raw_data.get(\"Kirche / Saal\")\n location[\"address\"] = raw_data.get(\"Bezirk\")\n" } ] } }, { "catalogue_id": "S035", "source_key": "orgelsite", "name": "Orgelsite NL", "selection_role": "registered_source", "parent_source_key": null, "attribution": "de Orgelsite / orgelsite.nl", "scope_description": "International; Dutch, German and English introductory text", "documentary_unit": "Organ webpage with photographs and search tags", "source_description": "The site reports approximately 35,000 organs and offers searches by place, builder, year and manual count. It accepts contributed photographs and explicitly reserves copyright in photographs, with exceptions identified for Wikimedia and YouTube material.", "source_description_references": [ "orgelsite", "record_orgelsite" ], "interpretation": "The approximate total is not an independently enumerated corpus. Tags aid discovery but should not be mistaken for controlled authority records. Photograph authorship needs item-level treatment.", "rights_observation": "The page requires prior written permission from the webmaster and photographer for republication of copyrighted photographs.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgelsite", "record_orgelsite" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgelsite", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Tagged organ article with contributed photography, historical prose and specification", "repeated_information": "Narrative chronology followed by division-based stop paragraphs and coupling list.", "date_and_state_distinctions": "Historical instrument years, photo years and publication/update dates are different attributes.", "interpretive_limit": "The named author of the specification can differ from the page author and photographers.", "observations": [ "Place, church and region are identified.", "Successive builders and restorers are named.", "Construction, replacement and restoration chronology.", "Stops are grouped in division paragraphs.", "Division ranges and note counts are supplied.", "Mechanical action and slider chests are described.", "A dated re-leathering of reservoir bellows is recorded.", "Case alteration and added performance controls are described.", "Surviving older case is distinguished from the replacement instrument.", "Multiple photographs have named authors and dates.", "The specification is attributed to a named contributor and website.", "Contributor website and taxonomic tags are linked.", "Page author, publication/update dates and photo credits are separate." ], "uninspected_or_unfilled_material": [], "narrative": "The Dordrecht article combines photographs, a dated organ history and specifications grouped by division. It distinguishes a surviving earlier case from the replacement instrument and records later work. The credited specification contributor, page author and photographers need not be the same person. Publication, update and photograph dates therefore serve different purposes." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The subclass accepts single-segment detail paths on its configured host and excludes category, tag, author, feed and media paths. This discovery rule is narrower than all pages on the website.", "transformation": "The transform uses taxonomy tags to identify a year, builder label and certain country labels. It suppresses implausible cached stop counts, retains the tags and records the parsed stop count. It otherwise uses the static HTML metadata, text and stoplist routines.", "limitations": "Tags may describe classification rather than a complete chronology. A calculated number of parsed rows must not be cited as a source-declared total. Detail discovery and tag vocabulary are code assumptions that may miss valid pages or differently labelled information." }, "implementation": { "class": "OrgelsiteNlCrawler", "class_line": 998, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrgelsiteNlCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orgelsite", "path": "aggregator/static_html_crawler.py", "line": 999, "class": "OrgelsiteNlCrawler" }, "id_column": { "value": "orgelsite_id", "path": "aggregator/static_html_crawler.py", "line": 1000, "class": "OrgelsiteNlCrawler" }, "default_country": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 28, "class": "StaticHtmlOrganCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://www.orgelsite.nl/" ], "path": "aggregator/static_html_crawler.py", "line": 1003, "class": "OrgelsiteNlCrawler" }, "detail_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 31, "class": "StaticHtmlOrganCrawler" }, "exclude_url_patterns": { "value": [ "/categorie/", "/tag/", "/author/", "/feed/", "/wp-content/", "mailto:", "#" ], "path": "aggregator/static_html_crawler.py", "line": 1004, "class": "OrgelsiteNlCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://www.orgelsite.nl", "path": "aggregator/static_html_crawler.py", "line": 1002, "class": "OrgelsiteNlCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelsiteNlCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 1014, "line_end": 1021, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 202, "line_end": 211, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 205, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 1023, "line_end": 1049, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_first_int" ], "literal_get_calls": [ { "line": 1038, "receiver": "transformed", "key": "specifications", "expression": "transformed.get('specifications', {})" }, { "line": 1033, "receiver": "prepared", "key": "stop_count", "expression": "prepared.get('stop_count')" }, { "line": 1039, "receiver": "specs", "key": "stops", "expression": "specs.get('stops', [])" }, { "line": 1041, "receiver": "specs", "key": "stop_count", "expression": "specs.get('stop_count')" }, { "line": 1045, "receiver": "location", "key": "country", "expression": "location.get('country')" }, { "line": 1024, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 1044, "receiver": "transformed", "key": "locations", "expression": "transformed.get('locations')" } ], "html_queries": [ { "line": 1025, "expression": "soup.select(\"a[rel='tag'], .tags-links a, .cat-links a\")" } ], "dictionary_entries": [], "subscript_assignments": [ { "line": 1037, "value_line": 1037, "key": "tags", "target_expression": "transformed['extensions'].setdefault(self.source_key, {})['tags']", "value_expression": "[tag for tag in tags if tag]" }, { "line": 1040, "value_line": 1040, "key": "parsed_stop_count", "target_expression": "specs['parsed_stop_count']", "value_expression": "parsed_stop_count" }, { "line": 1030, "value_line": 1030, "key": "year", "target_expression": "prepared['year']", "value_expression": "tag_years[0]" }, { "line": 1032, "value_line": 1032, "key": "builder", "target_expression": "prepared['builder']", "value_expression": "re.sub('\\\\s*\\\\(orgelbouwer\\\\)\\\\s*$', '', builder_tag, flags=re.I)" }, { "line": 1035, "value_line": 1035, "key": "stop_count", "target_expression": "prepared['stop_count']", "value_expression": "''" }, { "line": 1043, "value_line": 1043, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "parsed_stop_count" }, { "line": 1048, "value_line": 1048, "key": "country", "target_expression": "location['country']", "value_expression": "country" } ] } ], "selected_rule_rows": [ { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1035, "value_line": 1035, "key": "stop_count", "target_expression": "prepared['stop_count']", "value_expression": "''" }, { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 1048, "value_line": 1048, "key": "country", "target_expression": "location['country']", "value_expression": "country" }, { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "specifications", "value_expression": "transformed.get('specifications', {})", "line": 1038, "kind": "input read" }, { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "stops", "value_expression": "specs.get('stops', [])", "line": 1039, "kind": "input read" }, { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "html", "value_expression": "raw_data.get('html')", "line": 1024, "kind": "input read" }, { "symbol": "OrgelsiteNlCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "locations", "value_expression": "transformed.get('locations')", "line": 1044, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelsiteNlCrawler.transform", "line_start": 1032, "line_end": 1040, "text": " prepared[\"builder\"] = re.sub(r\"\\s*\\(orgelbouwer\\)\\s*$\", \"\", builder_tag, flags=re.I)\n cached_stop_count = self._first_int(prepared.get(\"stop_count\"))\n if cached_stop_count and (cached_stop_count > 500 or str(cached_stop_count) in tag_years):\n prepared[\"stop_count\"] = \"\"\n transformed = super().transform(prepared)\n transformed[\"extensions\"].setdefault(self.source_key, {})[\"tags\"] = [tag for tag in tags if tag]\n specs = transformed.get(\"specifications\", {})\n parsed_stop_count = len(specs.get(\"stops\", []))\n specs[\"parsed_stop_count\"] = parsed_stop_count" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelsiteNlCrawler.transform", "line_start": 1045, "line_end": 1049, "text": " if not location.get(\"country\"):\n country = next((tag for tag in tags if tag in {\"Nederland\", \"Belgie\", \"Duitsland\", \"Frankrijk\", \"Engeland\"} or tag.startswith(\"Engeland\")), \"\")\n if country:\n location[\"country\"] = country\n return transformed" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelsiteNlCrawler.transform", "line_start": 1035, "line_end": 1043, "text": " prepared[\"stop_count\"] = \"\"\n transformed = super().transform(prepared)\n transformed[\"extensions\"].setdefault(self.source_key, {})[\"tags\"] = [tag for tag in tags if tag]\n specs = transformed.get(\"specifications\", {})\n parsed_stop_count = len(specs.get(\"stops\", []))\n specs[\"parsed_stop_count\"] = parsed_stop_count\n mapped_stop_count = self._first_int(specs.get(\"stop_count\"))\n if parsed_stop_count and (not mapped_stop_count or mapped_stop_count > 500 or str(mapped_stop_count) in tag_years):\n specs[\"stop_count\"] = parsed_stop_count" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelsiteNlCrawler.transform", "line_start": 1036, "line_end": 1044, "text": " transformed = super().transform(prepared)\n transformed[\"extensions\"].setdefault(self.source_key, {})[\"tags\"] = [tag for tag in tags if tag]\n specs = transformed.get(\"specifications\", {})\n parsed_stop_count = len(specs.get(\"stops\", []))\n specs[\"parsed_stop_count\"] = parsed_stop_count\n mapped_stop_count = self._first_int(specs.get(\"stop_count\"))\n if parsed_stop_count and (not mapped_stop_count or mapped_stop_count > 500 or str(mapped_stop_count) in tag_years):\n specs[\"stop_count\"] = parsed_stop_count\n location = (transformed.get(\"locations\") or [{}])[0]" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelsiteNlCrawler.transform", "line_start": 1023, "line_end": 1029, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n soup = BeautifulSoup(raw_data.get(\"html\") or \"\", \"html.parser\")\n tags = [self._clean_text(link.get_text(\" \", strip=True)) for link in soup.select(\"a[rel='tag'], .tags-links a, .cat-links a\")]\n prepared = dict(raw_data)\n tag_years = [tag for tag in tags if re.fullmatch(r\"(?:1[5-9]\\d{2}|20\\d{2})\", tag)]\n builder_tag = next((tag for tag in tags if \"orgelbouwer\" in tag.lower()), \"\")\n if tag_years:" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelsiteNlCrawler.transform", "line_start": 1041, "line_end": 1049, "text": " mapped_stop_count = self._first_int(specs.get(\"stop_count\"))\n if parsed_stop_count and (not mapped_stop_count or mapped_stop_count > 500 or str(mapped_stop_count) in tag_years):\n specs[\"stop_count\"] = parsed_stop_count\n location = (transformed.get(\"locations\") or [{}])[0]\n if not location.get(\"country\"):\n country = next((tag for tag in tags if tag in {\"Nederland\", \"Belgie\", \"Duitsland\", \"Frankrijk\", \"Engeland\"} or tag.startswith(\"Engeland\")), \"\")\n if country:\n location[\"country\"] = country\n return transformed" } ] } }, { "catalogue_id": "S036", "source_key": "orgelverzeichnis_ch", "name": "Orgelverzeichnis Schweiz / Liechtenstein", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Peter Fasler; François Comment / Orgelverzeichnis Schweiz", "scope_description": "Switzerland and Liechtenstein", "documentary_unit": "Organ profile with dated specification", "source_description": "The rights statement credits information to congregations, builders, archives and contributors, with much fieldwork undertaken by Peter Fasler and François Comment. Individual profiles identify their information or image suppliers.", "source_description_references": [ "orgelverzeichnis_ch", "swiss_rights", "record_orgelverzeichnis_ch" ], "interpretation": "Preserve the profile's state date and the contributor attached to a specification. The legacy MAGIX address in Aggregator's investigation audit is an earlier location, not a second independent source.", "rights_observation": "The policy permits specified non-commercial uses, requires visible source and item credits, and requires permission for commercial use. It also preserves rights in contributed images.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orgelverzeichnis_ch", "swiss_rights", "record_orgelverzeichnis_ch" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orgelverzeichnis_ch", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Compact static profile with technical summary, specification and dated narrative", "repeated_information": "Label-value technical fields followed by stop rows and playing aids.", "date_and_state_distinctions": "Construction year and documentary stand date are separate.", "interpretive_limit": "Supplier credits for information and images are distinct; the stand date limits claims of currency.", "observations": [ "Institution, locality and canton are given.", "Builder and build year are populated.", "The construction is described in a dated note.", "Three named stops with pitches are tabulated.", "Manual compass is given.", "Separate action and windchest attributes.", "A transposition mechanism is described with two reference pitches.", "Retractable keyboard and transposition device are listed.", "Photograph is credited to an organ builder.", "Information supplier is named.", "Links connect regional lists, builders and HSLU.", "Information reference date and separate information/image credits." ], "uninspected_or_unfilled_material": [], "narrative": "The Aarau profile supplies a compact technical summary and three-stop specification with keyboard range and a transposition arrangement. An information reference date and separate information and image credits are given. That date limits a claim about currency; the photograph supplier is not automatically the author of the specification." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The index parser reads links and layout-dependent row context, including positioning information. The transform can override town, building and region with the acquired index values.", "transformation": "Detail extraction otherwise inherits the common static-page implementation. The index and page are therefore two evidential layers. Later source-specific Swiss date and specification-account helpers are separate from this inherited transform and must be cited separately when their results are used.", "limitations": "An information-reference date belongs to the specification account it labels, not automatically the instrument's construction. Layout-derived rows may fail under a changed presentation. The initial transfer should not be credited with all later offline evidence recovery." }, "implementation": { "class": "OrgelverzeichnisChCrawler", "class_line": 3423, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrgelverzeichnisChCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orgelverzeichnis_ch", "path": "aggregator/static_html_crawler.py", "line": 3424, "class": "OrgelverzeichnisChCrawler" }, "id_column": { "value": "orgelverzeichnis_ch_id", "path": "aggregator/static_html_crawler.py", "line": 3425, "class": "OrgelverzeichnisChCrawler" }, "default_country": { "value": "Switzerland", "path": "aggregator/static_html_crawler.py", "line": 3428, "class": "OrgelverzeichnisChCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://orgelverzeichnis.ch/Orgelverzeichnis/auswahl_kt.htm" ], "path": "aggregator/static_html_crawler.py", "line": 3429, "class": "OrgelverzeichnisChCrawler" }, "detail_url_patterns": { "value": [ "profile/" ], "path": "aggregator/static_html_crawler.py", "line": 3430, "class": "OrgelverzeichnisChCrawler" }, "exclude_url_patterns": { "value": [ "index.htm", "auswahl", "instr", "urheber", "quellen", "dank", ".pdf" ], "path": "aggregator/static_html_crawler.py", "line": 3431, "class": "OrgelverzeichnisChCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://orgelverzeichnis.ch", "path": "aggregator/static_html_crawler.py", "line": 3427, "class": "OrgelverzeichnisChCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler._canton_index_urls", "path": "aggregator/static_html_crawler.py", "line_start": 3499, "line_end": 3506, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 3502, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 3433, "line_end": 3437, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 3434, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 3593, "line_end": 3600, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 3594, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 3594, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 3595, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town', '')" }, { "line": 3595, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 3508, "line_end": 3537, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_dedupe_refs", "_identifier_from_url", "_is_detail_url", "_manuals_from_row", "_reference_from_parts", "_row_parts_for_link", "_stops_from_row", "_title_from_url", "_town_region" ], "literal_get_calls": [], "html_queries": [ { "line": 3511, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 3533, "key": "index_url", "value_expression": "page_url" }, { "line": 3533, "key": "row", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler._stops_from_row", "path": "aggregator/static_html_crawler.py", "line_start": 3587, "line_end": 3591, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 3439, "line_end": 3497, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_canton_index_urls", "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 3440, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 3441, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 3442, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 3486, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" }, { "line": 3470, "receiver": "ref", "key": "url", "expression": "ref.get('url', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrgelverzeichnisChCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 3539, "line_end": 3548, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 3542, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 3544, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 3546, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 3541, "receiver": "transformed", "key": "locations", "expression": "transformed.get('locations')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 3543, "value_line": 3543, "key": "town", "target_expression": "location['town']", "value_expression": "raw_data['town']" }, { "line": 3545, "value_line": 3545, "key": "building", "target_expression": "location['building']", "value_expression": "raw_data['building']" }, { "line": 3547, "value_line": 3547, "key": "region", "target_expression": "location['region']", "value_expression": "raw_data['region']" } ] } ], "selected_rule_rows": [ { "symbol": "OrgelverzeichnisChCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "town", "value_expression": "raw_data.get('town')", "line": 3542, "kind": "input read" }, { "symbol": "OrgelverzeichnisChCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "building", "value_expression": "raw_data.get('building')", "line": 3544, "kind": "input read" }, { "symbol": "OrgelverzeichnisChCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "region", "value_expression": "raw_data.get('region')", "line": 3546, "kind": "input read" }, { "symbol": "OrgelverzeichnisChCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "locations", "value_expression": "transformed.get('locations')", "line": 3541, "kind": "input read" }, { "symbol": "OrgelverzeichnisChCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "key": "timeout", "value_expression": "self.config.get('timeout', 30)", "line": 3434, "kind": "input read" }, { "symbol": "OrgelverzeichnisChCrawler.crawl", "path": "aggregator/static_html_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 3440, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelverzeichnisChCrawler.transform", "line_start": 3539, "line_end": 3547, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n transformed = super().transform(raw_data)\n location = (transformed.get(\"locations\") or [{}])[0]\n if raw_data.get(\"town\"):\n location[\"town\"] = raw_data[\"town\"]\n if raw_data.get(\"building\"):\n location[\"building\"] = raw_data[\"building\"]\n if raw_data.get(\"region\"):\n location[\"region\"] = raw_data[\"region\"]" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelverzeichnisChCrawler.transform", "line_start": 3541, "line_end": 3548, "text": " location = (transformed.get(\"locations\") or [{}])[0]\n if raw_data.get(\"town\"):\n location[\"town\"] = raw_data[\"town\"]\n if raw_data.get(\"building\"):\n location[\"building\"] = raw_data[\"building\"]\n if raw_data.get(\"region\"):\n location[\"region\"] = raw_data[\"region\"]\n return transformed" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelverzeichnisChCrawler.transform", "line_start": 3543, "line_end": 3548, "text": " location[\"town\"] = raw_data[\"town\"]\n if raw_data.get(\"building\"):\n location[\"building\"] = raw_data[\"building\"]\n if raw_data.get(\"region\"):\n location[\"region\"] = raw_data[\"region\"]\n return transformed" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelverzeichnisChCrawler.transform", "line_start": 3539, "line_end": 3546, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n transformed = super().transform(raw_data)\n location = (transformed.get(\"locations\") or [{}])[0]\n if raw_data.get(\"town\"):\n location[\"town\"] = raw_data[\"town\"]\n if raw_data.get(\"building\"):\n location[\"building\"] = raw_data[\"building\"]\n if raw_data.get(\"region\"):" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelverzeichnisChCrawler._fetch_html", "line_start": 3433, "line_end": 3437, "text": " def _fetch_html(self, url: str) -> str:\n response = self.session.get(url, timeout=int(self.config.get(\"timeout\", 30)))\n response.raise_for_status()\n encoding = response.apparent_encoding or response.encoding or \"utf-8\"\n return response.content.decode(encoding, errors=\"ignore\")" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OrgelverzeichnisChCrawler.crawl", "line_start": 3439, "line_end": 3445, "text": " def crawl(self, **kwargs) -> List[Dict[str, Any]]:\n task_id = kwargs.get(\"task_id\") or f\"{self.source_key}_crawl_{datetime.now().strftime('%Y%m%d%H%M%S')}\"\n max_items = self._optional_int(kwargs.get(\"max_items\"))\n start_urls = kwargs.get(\"start_urls\") or self.start_urls\n if isinstance(start_urls, str):\n start_urls = [item.strip() for item in start_urls.split(\",\") if item.strip()]\n" } ] } }, { "catalogue_id": "S037", "source_key": "orguesch", "name": "OrguesCH", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Orgues.ch", "scope_description": "Selected Swiss construction, restoration and revision projects", "documentary_unit": "Project or instrument presentation", "source_description": "The current home page describes organ consultancy and coordination and presents nearly 60 instruments built, restored or revised with partner builders. It also advertises other activities, including Allen Organs representation.", "source_description_references": [ "orguesch", "record_orguesch" ], "interpretation": "This is a selected project portfolio rather than a Swiss national inventory. Its commercial and collaborative context belongs in the source description. The current presentation should not inherit the broader implications of an older generic database label.", "rights_observation": "The current page states that all rights are reserved.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orguesch", "record_orguesch" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orguesch", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "access_finding": "The inherited detail route returned an organ-not-found message despite HTTP 200. This does not establish that the portfolio or instrument has disappeared." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Index cards are selected through organ-detail URL patterns. Titles are sought in headings, image alternative text, styled paragraphs and finally link text; the last paragraph can supply town context.", "transformation": "The subclass supplies card discovery and a Switzerland default. Detailed extraction otherwise inherits the static HTML transform. That shared route preserves main text, metadata, table rows, images and links while attempting a labelled specification and a builder/year summary.", "limitations": "The presence of a structured card does not imply that the detail page has the same metadata model. Country defaults and inherited first-year/construction logic are interpretive operations. Source text retained outside structured fields should be examined before declaring information absent." }, "implementation": { "class": "OrguesChCrawler", "class_line": 1052, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrguesChCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orguesch", "path": "aggregator/static_html_crawler.py", "line": 1053, "class": "OrguesChCrawler" }, "id_column": { "value": "orguesch_id", "path": "aggregator/static_html_crawler.py", "line": 1054, "class": "OrguesChCrawler" }, "default_country": { "value": "Switzerland", "path": "aggregator/static_html_crawler.py", "line": 1057, "class": "OrguesChCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://www.orgues.ch/orgues" ], "path": "aggregator/static_html_crawler.py", "line": 1058, "class": "OrguesChCrawler" }, "detail_url_patterns": { "value": [ "/orgues/orgue-" ], "path": "aggregator/static_html_crawler.py", "line": 1059, "class": "OrguesChCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://www.orgues.ch", "path": "aggregator/static_html_crawler.py", "line": 1056, "class": "OrguesChCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrguesChCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 1061, "line_end": 1076, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [ { "line": 1065, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 1068, "receiver": "link.find('img')", "key": "alt", "expression": "link.find('img').get('alt')" } ], "html_queries": [ { "line": 1064, "expression": "soup.select(\"a[href^='/orgues/orgue-'], a[href*='/orgues/orgue-']\")" }, { "line": 1066, "expression": "link.find('h3')" }, { "line": 1067, "expression": "link.find('img')" }, { "line": 1069, "expression": "link.find('p', class_=re.compile('font-medium'))" }, { "line": 1073, "expression": "link.find_all('p')" }, { "line": 1066, "expression": "link.find('h3')" }, { "line": 1070, "expression": "link.find('p', class_=re.compile('font-medium'))" }, { "line": 1068, "expression": "link.find('img')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 434, "line_end": 516, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_extract_images", "_extract_links", "_extract_metadata", "_extract_raw_spec", "_extract_table_rows", "_extract_title", "_first_int", "_first_metadata", "_first_year", "_location_parts", "_main_text", "_safe_json_loads" ], "literal_get_calls": [ { "line": 435, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 440, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 441, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 442, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 443, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 444, "receiver": "raw_data", "key": "rank_count", "expression": "raw_data.get('rank_count')" }, { "line": 513, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 442, "receiver": "metadata", "key": "manuals", "expression": "metadata.get('manuals')" }, { "line": 442, "receiver": "metadata", "key": "manual", "expression": "metadata.get('manual')" }, { "line": 443, "receiver": "metadata", "key": "stops", "expression": "metadata.get('stops')" }, { "line": 443, "receiver": "metadata", "key": "stop", "expression": "metadata.get('stop')" }, { "line": 444, "receiver": "metadata", "key": "ranks", "expression": "metadata.get('ranks')" }, { "line": 444, "receiver": "metadata", "key": "rank", "expression": "metadata.get('rank')" }, { "line": 480, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 453, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 454, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 479, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 487, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': raw_data.get('country') or self.default_country, 'region': raw_data.get('region') or self.default_region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 460, "key": "involved", "value_expression": "self._build_involved(builder, year)" }, { "line": 461, "key": "events", "value_expression": "[{'type': 'construction', 'date': str(year), 'description': builder}] if year else []" }, { "line": 462, "key": "specifications", "value_expression": "{'manual_count': manual_count, 'stop_count': stop_count, 'rank_count': rank_count, 'stops_by_division': stops_by_division, 'stops': [{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops], 'raw_spec': raw_spec, 'technical_rows_unstructured': [{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows}" }, { "line": 479, "key": "media", "value_expression": "{'images': self._extract_images(soup, raw_data.get('url') or '')}" }, { "line": 480, "key": "links", "value_expression": "self._extract_links(soup, raw_data.get('url') or '')" }, { "line": 481, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': text}] if text else []" }, { "line": 482, "key": "other_identifiers", "value_expression": "[{'type': self.id_column, 'value': raw_data.get(self.id_column)}]" }, { "line": 483, "key": "extensions", "value_expression": "{self.source_key: {'title': title, 'metadata': metadata, 'raw_data': self._safe_json_loads(raw_data.get('raw_data_json'), {}), 'reference_context': {key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}}}" }, { "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 467, "key": "stops", "value_expression": "[{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 472, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 473, "key": "technical_rows_unstructured", "value_expression": "[{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows" }, { "line": 479, "key": "images", "value_expression": "self._extract_images(soup, raw_data.get('url') or '')" }, { "line": 452, "key": "status", "value_expression": "'current_location'" }, { "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "line": 454, "key": "region", "value_expression": "raw_data.get('region') or self.default_region" }, { "line": 455, "key": "town", "value_expression": "town" }, { "line": 456, "key": "building", "value_expression": "building" }, { "line": 457, "key": "raw_str", "value_expression": "title" }, { "line": 482, "key": "type", "value_expression": "self.id_column" }, { "line": 482, "key": "value", "value_expression": "raw_data.get(self.id_column)" }, { "line": 485, "key": "title", "value_expression": "title" }, { "line": 486, "key": "metadata", "value_expression": "metadata" }, { "line": 487, "key": "raw_data", "value_expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {})" }, { "line": 488, "key": "reference_context", "value_expression": "{key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}" }, { "line": 461, "key": "type", "value_expression": "'construction'" }, { "line": 461, "key": "date", "value_expression": "str(year)" }, { "line": 461, "key": "description", "value_expression": "builder" }, { "line": 468, "key": "name", "value_expression": "stop" }, { "line": 468, "key": "division", "value_expression": "division" }, { "line": 468, "key": "raw_str", "value_expression": "stop" }, { "line": 481, "key": "type", "value_expression": "'source_text'" }, { "line": 481, "key": "text", "value_expression": "text" }, { "line": 474, "key": "label", "value_expression": "key" }, { "line": 474, "key": "value", "value_expression": "value" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 457, "key": "raw_str", "value_expression": "title" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 461, "line_end": 469, "text": " \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 460, "line_end": 468, "text": " \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 462, "line_end": 470, "text": " \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 463, "line_end": 471, "text": " \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 450, "line_end": 458, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": raw_data.get(\"country\") or self.default_country,\n \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 454, "line_end": 462, "text": " \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }\n ],\n \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {" } ] } }, { "catalogue_id": "S038", "source_key": "orguesfrance", "name": "Orgues France", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Orgues en France et dans le monde", "scope_description": "France and other countries", "documentary_unit": "Compiled organ profile", "source_description": "The personal site's introduction says that it has collected information from the web since 2008. On 10 September 2026 it reported more than 3,700 pipe organs in 38 countries. It also offers multilingual stop-name and organ-term aids.", "source_description_references": [ "orguesfrance", "record_orguesfrance" ], "interpretation": "Its explicit compilation from other websites makes source dependence a central issue. Agreement between compiled websites does not, by itself, establish independent corroboration; the cited antecedents must be examined. The two Orguesfrance entries in the investigation audit are one registered source.", "rights_observation": "No general open licence was established from the consulted home page.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "orguesfrance", "record_orguesfrance" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_orguesfrance", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Static compiled organ profile with metadata block, narrative and linked specification/media", "repeated_information": "Geographic/building/builder fields precede history and technical material.", "date_and_state_distinctions": "Page creation/update dates are separate from instrument construction and relocation events.", "interpretive_limit": "The profile discusses predecessor instruments in the same institution. Media and specification may be separate linked assets.", "observations": [ "Country, region, department, city and venue are labelled.", "Builder and construction year are labelled.", "A dated account covers predecessors and the new concert organ.", "Different actions of the two consoles are described.", "Variable wind pressure and high-pressure stops are discussed.", "Two-console arrangement and performance controls are described.", "Map, photograph, specification and video links are supplied.", "Page creation and update dates are explicit." ], "uninspected_or_unfilled_material": [ "External video presentations are linked; their contents were not inspected.", "Specification is offered as linked material; a text stoplist was not observed in this response.", "Photograph links are present; their contents were not separately inspected." ], "narrative": "The Radio France profile combines labelled location and builder information with a historical narrative that includes predecessor instruments. Technical discussion describes two consoles and wind arrangements. Specifications, photographs and videos are offered through links whose contents were not separately inspected. Page creation and update dates are distinct from the events described in the organ history." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The subclass implements source-specific crawling and index parsing before passing detail pages to the inherited transform.", "transformation": "The inherited implementation collects main text, label/value metadata, selected stoplist text, table rows, images and links. It projects a builder/year summary and source-reference context. These are implemented operations; they do not establish a formal native database schema behind the website.", "limitations": "The configured crawl route bounds what is discovered. A narrative can contain several configurations even when the output has one principal specification. Generic label and date heuristics require a record-level check before their results support historical comparisons." }, "implementation": { "class": "OrguesFranceCrawler", "class_line": 1970, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OrguesFranceCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "orguesfrance", "path": "aggregator/static_html_crawler.py", "line": 1971, "class": "OrguesFranceCrawler" }, "id_column": { "value": "orguesfrance_id", "path": "aggregator/static_html_crawler.py", "line": 1972, "class": "OrguesFranceCrawler" }, "default_country": { "value": "France", "path": "aggregator/static_html_crawler.py", "line": 1975, "class": "OrguesFranceCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "http://orguesfrance.com/Select.php" ], "path": "aggregator/static_html_crawler.py", "line": 1976, "class": "OrguesFranceCrawler" }, "detail_url_patterns": { "value": [ "orguesfrance.com/" ], "path": "aggregator/static_html_crawler.py", "line": 1977, "class": "OrguesFranceCrawler" }, "exclude_url_patterns": { "value": [ "select", "index.html", ".jpg", ".png", ".gif" ], "path": "aggregator/static_html_crawler.py", "line": 1978, "class": "OrguesFranceCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "http://orguesfrance.com", "path": "aggregator/static_html_crawler.py", "line": 1974, "class": "OrguesFranceCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrguesFranceCrawler._fetch_search_results", "path": "aggregator/static_html_crawler.py", "line_start": 2016, "line_end": 2032, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 2022, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 2029, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 2017, "key": "Referer", "value_expression": "urljoin(self.base_url, '/Select.php')" }, { "line": 2027, "key": "Vville", "value_expression": "city" }, { "line": 2027, "key": "Llocal", "value_expression": "locality" }, { "line": 2027, "key": "Ffacteur", "value_expression": "builder" }, { "line": 2027, "key": "afficher", "value_expression": "'ok'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrguesFranceCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 2034, "line_end": 2066, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_result_table" ], "literal_get_calls": [], "html_queries": [ { "line": 2040, "expression": "table.find_all('tr')" }, { "line": 2042, "expression": "row.find('a', href=True)" }, { "line": 2041, "expression": "row.find_all(['td', 'th'], recursive=False)" } ], "dictionary_entries": [ { "line": 2061, "key": "created", "value_expression": "cells[8] if len(cells) > 8 else ''" }, { "line": 2062, "key": "modified", "value_expression": "cells[9] if len(cells) > 9 else ''" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OrguesFranceCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 1980, "line_end": 2014, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_search_results", "_optional_int", "_parse_index_page", "_save_reference", "_truthy", "_update_scrape_status" ], "literal_get_calls": [ { "line": 1981, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 1982, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 1983, "receiver": "kwargs", "key": "ville", "expression": "kwargs.get('ville')" }, { "line": 1985, "receiver": "kwargs", "key": "locality", "expression": "kwargs.get('locality')" }, { "line": 1985, "receiver": "kwargs", "key": "localisation", "expression": "kwargs.get('localisation')" }, { "line": 1986, "receiver": "kwargs", "key": "builder", "expression": "kwargs.get('builder')" }, { "line": 1986, "receiver": "kwargs", "key": "facteur", "expression": "kwargs.get('facteur')" }, { "line": 1984, "receiver": "self.config", "key": "default_city", "expression": "self.config.get('default_city')" }, { "line": 1987, "receiver": "kwargs", "key": "all_records", "expression": "kwargs.get('all_records')" }, { "line": 1987, "receiver": "kwargs", "key": "all", "expression": "kwargs.get('all')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 434, "line_end": 516, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_extract_images", "_extract_links", "_extract_metadata", "_extract_raw_spec", "_extract_table_rows", "_extract_title", "_first_int", "_first_metadata", "_first_year", "_location_parts", "_main_text", "_safe_json_loads" ], "literal_get_calls": [ { "line": 435, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 440, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 441, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 442, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 443, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 444, "receiver": "raw_data", "key": "rank_count", "expression": "raw_data.get('rank_count')" }, { "line": 513, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 442, "receiver": "metadata", "key": "manuals", "expression": "metadata.get('manuals')" }, { "line": 442, "receiver": "metadata", "key": "manual", "expression": "metadata.get('manual')" }, { "line": 443, "receiver": "metadata", "key": "stops", "expression": "metadata.get('stops')" }, { "line": 443, "receiver": "metadata", "key": "stop", "expression": "metadata.get('stop')" }, { "line": 444, "receiver": "metadata", "key": "ranks", "expression": "metadata.get('ranks')" }, { "line": 444, "receiver": "metadata", "key": "rank", "expression": "metadata.get('rank')" }, { "line": 480, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 453, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 454, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 479, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 487, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': raw_data.get('country') or self.default_country, 'region': raw_data.get('region') or self.default_region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 460, "key": "involved", "value_expression": "self._build_involved(builder, year)" }, { "line": 461, "key": "events", "value_expression": "[{'type': 'construction', 'date': str(year), 'description': builder}] if year else []" }, { "line": 462, "key": "specifications", "value_expression": "{'manual_count': manual_count, 'stop_count': stop_count, 'rank_count': rank_count, 'stops_by_division': stops_by_division, 'stops': [{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops], 'raw_spec': raw_spec, 'technical_rows_unstructured': [{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows}" }, { "line": 479, "key": "media", "value_expression": "{'images': self._extract_images(soup, raw_data.get('url') or '')}" }, { "line": 480, "key": "links", "value_expression": "self._extract_links(soup, raw_data.get('url') or '')" }, { "line": 481, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': text}] if text else []" }, { "line": 482, "key": "other_identifiers", "value_expression": "[{'type': self.id_column, 'value': raw_data.get(self.id_column)}]" }, { "line": 483, "key": "extensions", "value_expression": "{self.source_key: {'title': title, 'metadata': metadata, 'raw_data': self._safe_json_loads(raw_data.get('raw_data_json'), {}), 'reference_context': {key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}}}" }, { "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 467, "key": "stops", "value_expression": "[{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 472, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 473, "key": "technical_rows_unstructured", "value_expression": "[{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows" }, { "line": 479, "key": "images", "value_expression": "self._extract_images(soup, raw_data.get('url') or '')" }, { "line": 452, "key": "status", "value_expression": "'current_location'" }, { "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "line": 454, "key": "region", "value_expression": "raw_data.get('region') or self.default_region" }, { "line": 455, "key": "town", "value_expression": "town" }, { "line": 456, "key": "building", "value_expression": "building" }, { "line": 457, "key": "raw_str", "value_expression": "title" }, { "line": 482, "key": "type", "value_expression": "self.id_column" }, { "line": 482, "key": "value", "value_expression": "raw_data.get(self.id_column)" }, { "line": 485, "key": "title", "value_expression": "title" }, { "line": 486, "key": "metadata", "value_expression": "metadata" }, { "line": 487, "key": "raw_data", "value_expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {})" }, { "line": 488, "key": "reference_context", "value_expression": "{key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}" }, { "line": 461, "key": "type", "value_expression": "'construction'" }, { "line": 461, "key": "date", "value_expression": "str(year)" }, { "line": 461, "key": "description", "value_expression": "builder" }, { "line": 468, "key": "name", "value_expression": "stop" }, { "line": 468, "key": "division", "value_expression": "division" }, { "line": 468, "key": "raw_str", "value_expression": "stop" }, { "line": 481, "key": "type", "value_expression": "'source_text'" }, { "line": 481, "key": "text", "value_expression": "text" }, { "line": 474, "key": "label", "value_expression": "key" }, { "line": 474, "key": "value", "value_expression": "value" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 457, "key": "raw_str", "value_expression": "title" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 461, "line_end": 469, "text": " \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 460, "line_end": 468, "text": " \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 462, "line_end": 470, "text": " \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 463, "line_end": 471, "text": " \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 450, "line_end": 458, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": raw_data.get(\"country\") or self.default_country,\n \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 454, "line_end": 462, "text": " \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }\n ],\n \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {" } ] } }, { "catalogue_id": "S039", "source_key": "oswa", "name": "OSWA Pipe Organs in Western Australia", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Organ Society of Western Australia", "scope_description": "Western Australia", "documentary_unit": "Regional organ profile", "source_description": "The society describes preservation and advisory work on Western Australia's organ heritage and links to its pipe-organ directory. Its wider remit also includes electronic and hybrid instruments.", "source_description_references": [ "oswa", "record_oswa" ], "interpretation": "Aggregator's pipe-organ selection should be distinguished from the society's complete activities. State-level profiles can complement OHTA, but overlapping coverage must be checked at the instrument and documentary-source level.", "rights_observation": "The society displays an all-rights-reserved notice; no general reuse licence was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "oswa", "record_oswa" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_oswa", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Gazetteer form with property, previous-organ, present-organ and document-control blocks", "repeated_information": "Label-value fields, division stop rows and repeated dated work narratives.", "date_and_state_distinctions": "Previous organs, current organ, relocation dates and document-control dates are separate.", "interpretive_limit": "Many form labels are unfilled. The existence of a wind-pressure or heritage-number field does not establish a value.", "observations": [ "Institution, street, city, state, country and room fields.", "Named maker and completion/installation year.", "Original installation, relocation and later work are separately described.", "Division stop rows with pitch and total stops/ranks/pipes.", "Divisions and manual count are populated; key-compass fields are blank.", "Key and stop actions are separately populated.", "Placement, open casework and console configuration are supplied.", "The organ is described as in use and in good condition.", "Photographs have named contributors.", "Gazetteer editions are cited in document control.", "Comparable instrument and builder links.", "Document control identifies original entries, contributor and revision dates." ], "uninspected_or_unfilled_material": [ "A wind-pressure label is present without a value." ], "narrative": "The Albany entry uses a form with property, previous-organ, present-organ and document-control sections. It describes relocation and later work and gives a division specification. Some technical labels, including wind pressure and key compass, are unfilled. The form structure is evidence of what can be recorded; it is not evidence that every entry supplies a value." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The town index is parsed as a table: town, building, builder, date, manual, rank and stop counts occupy positional cells; an optional additional cell retains pipes. Detail pages are fetched separately.", "transformation": "The detail override prefers explicitly labelled builder, completion/installation date and total-stop fields. Specification extraction chooses the last leaf-table candidate containing a recognised division line. Declared, parsed and reference stop counts are kept separately, with declared count preferred for the main value.", "limitations": "Choosing the last candidate is an extraction rule, not independent evidence that it describes the latest organ. Parsed-row counts and declared stops are different quantities. Positional index columns must be checked when the source layout changes." }, "implementation": { "class": "OswaCrawler", "class_line": 912, "path": "aggregator/static_html_crawler.py", "inheritance": [ "OswaCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "oswa", "path": "aggregator/static_html_crawler.py", "line": 913, "class": "OswaCrawler" }, "id_column": { "value": "oswa_id", "path": "aggregator/static_html_crawler.py", "line": 914, "class": "OswaCrawler" }, "default_country": { "value": "Australia", "path": "aggregator/static_html_crawler.py", "line": 917, "class": "OswaCrawler" }, "default_region": { "value": "Western Australia", "path": "aggregator/static_html_crawler.py", "line": 918, "class": "OswaCrawler" }, "start_urls": { "value": [ "https://www.oswa.org.au/sup/WAOrgans/Indexes/WAOrganTownIndex.html" ], "path": "aggregator/static_html_crawler.py", "line": 919, "class": "OswaCrawler" }, "detail_url_patterns": { "value": [ "/sup/waorgans/" ], "path": "aggregator/static_html_crawler.py", "line": 920, "class": "OswaCrawler" }, "exclude_url_patterns": { "value": [ "/indexes/" ], "path": "aggregator/static_html_crawler.py", "line": 921, "class": "OswaCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://www.oswa.org.au", "path": "aggregator/static_html_crawler.py", "line": 916, "class": "OswaCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OswaCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 923, "line_end": 934, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 925, "expression": "soup.find_all('table')" }, { "line": 926, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OswaCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 961, "line_end": 995, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_reference_from_parts" ], "literal_get_calls": [], "html_queries": [ { "line": 964, "expression": "soup.find('table')" }, { "line": 967, "expression": "table.find_all('tr')" }, { "line": 968, "expression": "row.find_all(['td', 'th'], recursive=False)" }, { "line": 971, "expression": "row.find('a', href=True)" } ], "dictionary_entries": [ { "line": 991, "key": "pipes", "value_expression": "self._clean_text(cells[7].get_text(' ', strip=True)) if len(cells) > 7 else ''" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 936, "line_end": 959, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_first_int" ], "literal_get_calls": [ { "line": 951, "receiver": "transformed", "key": "specifications", "expression": "transformed.get('specifications', {})" }, { "line": 952, "receiver": "transformed.get('extensions', {}).get(self.source_key, {})", "key": "metadata", "expression": "transformed.get('extensions', {}).get(self.source_key, {}).get('metadata', {})" }, { "line": 947, "receiver": "exact_fields", "key": "builder's name", "expression": "exact_fields.get(\"builder's name\")" }, { "line": 947, "receiver": "prepared", "key": "builder", "expression": "prepared.get('builder', '')" }, { "line": 948, "receiver": "exact_fields", "key": "date of completion/installation", "expression": "exact_fields.get('date of completion/installation')" }, { "line": 948, "receiver": "prepared", "key": "year", "expression": "prepared.get('year', '')" }, { "line": 949, "receiver": "exact_fields", "key": "total number of stops", "expression": "exact_fields.get('total number of stops')" }, { "line": 949, "receiver": "prepared", "key": "stop_count", "expression": "prepared.get('stop_count', '')" }, { "line": 953, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 955, "receiver": "specs", "key": "stops", "expression": "specs.get('stops', [])" }, { "line": 937, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 937, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 954, "receiver": "exact_fields", "key": "total number of stops", "expression": "exact_fields.get('total number of stops')" }, { "line": 954, "receiver": "metadata", "key": "stops", "expression": "metadata.get('stops')" }, { "line": 952, "receiver": "transformed", "key": "extensions", "expression": "transformed.get('extensions', {})" } ], "html_queries": [ { "line": 940, "expression": "soup.find_all('tr')" }, { "line": 941, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [ { "line": 947, "value_line": 947, "key": "builder", "target_expression": "prepared['builder']", "value_expression": "exact_fields.get(\"builder's name\") or prepared.get('builder', '')" }, { "line": 948, "value_line": 948, "key": "year", "target_expression": "prepared['year']", "value_expression": "exact_fields.get('date of completion/installation') or prepared.get('year', '')" }, { "line": 949, "value_line": 949, "key": "stop_count", "target_expression": "prepared['stop_count']", "value_expression": "exact_fields.get('total number of stops') or prepared.get('stop_count', '')" }, { "line": 956, "value_line": 956, "key": "reference_stop_count", "target_expression": "specs['reference_stop_count']", "value_expression": "reference_count" }, { "line": 957, "value_line": 957, "key": "parsed_stop_count", "target_expression": "specs['parsed_stop_count']", "value_expression": "parsed_count" }, { "line": 958, "value_line": 958, "key": "stop_count", "target_expression": "specs['stop_count']", "value_expression": "declared_count or parsed_count or reference_count" } ] } ], "selected_rule_rows": [ { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 949, "value_line": 949, "key": "stop_count", "target_expression": "prepared['stop_count']", "value_expression": "exact_fields.get('total number of stops') or prepared.get('stop_count', '')" }, { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "specifications", "value_expression": "transformed.get('specifications', {})", "line": 951, "kind": "input read" }, { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "metadata", "value_expression": "transformed.get('extensions', {}).get(self.source_key, {}).get('metadata', {})", "line": 952, "kind": "input read" }, { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "builder's name", "value_expression": "exact_fields.get(\"builder's name\")", "line": 947, "kind": "input read" }, { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "builder", "value_expression": "prepared.get('builder', '')", "line": 947, "kind": "input read" }, { "symbol": "OswaCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "date of completion/installation", "value_expression": "exact_fields.get('date of completion/installation')", "line": 948, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "OswaCrawler.transform", "line_start": 946, "line_end": 954, "text": " exact_fields[label] = self._clean_text(cells[1].get_text(\" \", strip=True))\n prepared[\"builder\"] = exact_fields.get(\"builder's name\") or prepared.get(\"builder\", \"\")\n prepared[\"year\"] = exact_fields.get(\"date of completion/installation\") or prepared.get(\"year\", \"\")\n prepared[\"stop_count\"] = exact_fields.get(\"total number of stops\") or prepared.get(\"stop_count\", \"\")\n transformed = super().transform(prepared)\n specs = transformed.get(\"specifications\", {})\n metadata = transformed.get(\"extensions\", {}).get(self.source_key, {}).get(\"metadata\", {})\n reference_count = self._first_int(raw_data.get(\"stop_count\"))\n declared_count = self._first_int(exact_fields.get(\"total number of stops\")) or self._first_int(metadata.get(\"stops\"))" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OswaCrawler.transform", "line_start": 948, "line_end": 956, "text": " prepared[\"year\"] = exact_fields.get(\"date of completion/installation\") or prepared.get(\"year\", \"\")\n prepared[\"stop_count\"] = exact_fields.get(\"total number of stops\") or prepared.get(\"stop_count\", \"\")\n transformed = super().transform(prepared)\n specs = transformed.get(\"specifications\", {})\n metadata = transformed.get(\"extensions\", {}).get(self.source_key, {}).get(\"metadata\", {})\n reference_count = self._first_int(raw_data.get(\"stop_count\"))\n declared_count = self._first_int(exact_fields.get(\"total number of stops\")) or self._first_int(metadata.get(\"stops\"))\n parsed_count = len(specs.get(\"stops\", []))\n specs[\"reference_stop_count\"] = reference_count" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OswaCrawler.transform", "line_start": 949, "line_end": 957, "text": " prepared[\"stop_count\"] = exact_fields.get(\"total number of stops\") or prepared.get(\"stop_count\", \"\")\n transformed = super().transform(prepared)\n specs = transformed.get(\"specifications\", {})\n metadata = transformed.get(\"extensions\", {}).get(self.source_key, {}).get(\"metadata\", {})\n reference_count = self._first_int(raw_data.get(\"stop_count\"))\n declared_count = self._first_int(exact_fields.get(\"total number of stops\")) or self._first_int(metadata.get(\"stops\"))\n parsed_count = len(specs.get(\"stops\", []))\n specs[\"reference_stop_count\"] = reference_count\n specs[\"parsed_stop_count\"] = parsed_count" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OswaCrawler.transform", "line_start": 944, "line_end": 952, "text": " label = self._clean_label(cells[0].get_text(\" \", strip=True))\n if label in {\"builder's name\", \"date of completion/installation\", \"total number of stops\"}:\n exact_fields[label] = self._clean_text(cells[1].get_text(\" \", strip=True))\n prepared[\"builder\"] = exact_fields.get(\"builder's name\") or prepared.get(\"builder\", \"\")\n prepared[\"year\"] = exact_fields.get(\"date of completion/installation\") or prepared.get(\"year\", \"\")\n prepared[\"stop_count\"] = exact_fields.get(\"total number of stops\") or prepared.get(\"stop_count\", \"\")\n transformed = super().transform(prepared)\n specs = transformed.get(\"specifications\", {})\n metadata = transformed.get(\"extensions\", {}).get(self.source_key, {}).get(\"metadata\", {})" }, { "path": "aggregator/static_html_crawler.py", "symbol": "OswaCrawler.transform", "line_start": 945, "line_end": 953, "text": " if label in {\"builder's name\", \"date of completion/installation\", \"total number of stops\"}:\n exact_fields[label] = self._clean_text(cells[1].get_text(\" \", strip=True))\n prepared[\"builder\"] = exact_fields.get(\"builder's name\") or prepared.get(\"builder\", \"\")\n prepared[\"year\"] = exact_fields.get(\"date of completion/installation\") or prepared.get(\"year\", \"\")\n prepared[\"stop_count\"] = exact_fields.get(\"total number of stops\") or prepared.get(\"stop_count\", \"\")\n transformed = super().transform(prepared)\n specs = transformed.get(\"specifications\", {})\n metadata = transformed.get(\"extensions\", {}).get(self.source_key, {}).get(\"metadata\", {})\n reference_count = self._first_int(raw_data.get(\"stop_count\"))" } ] } }, { "catalogue_id": "S040", "source_key": "pcorgan_hauptwerk_samplesets", "name": "PCorgan Hauptwerk Sample Set Registry", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Gert van Ginkel / PCorgan", "scope_description": "Mainly church-organ sample sets for Hauptwerk", "documentary_unit": "Comparative sample-set table row", "source_description": "The comparison table records product, producer, style, size, indicative price and technical characteristics. It defines its memory column as a particular Hauptwerk-plus-set loading scenario. The author explains the site's church-organ focus and discloses receiving sets for review or beta testing.", "source_description_references": [ "pcorgan_hauptwerk_samplesets", "pcorgan_about" ], "interpretation": "The visible prices are explicitly indicative and may be outdated. Memory values are not interchangeable with download sizes or modern maximum-memory configurations. This source is a comparative index, not the producer's release record.", "rights_observation": "Copyright is asserted; permissions for linked audio and products remain with their respective sources.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "pcorgan_hauptwerk_samplesets", "pcorgan_about" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects catalogue list links and links matching the SampleSets pattern. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is third-party sample-set registry items. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A registry may summarise producer material; current technical details and rights require the cited producer edition. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "PcorganHauptwerkSamplesetsCrawler", "class_line": 1654, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "PcorganHauptwerkSamplesetsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "pcorgan_hauptwerk_samplesets", "path": "aggregator/grandorgue_library_crawler.py", "line": 1655, "class": "PcorganHauptwerkSamplesetsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "registry", "path": "aggregator/grandorgue_library_crawler.py", "line": 1657, "class": "PcorganHauptwerkSamplesetsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1659, "line_end": 1701, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1665, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 1663, "expression": "soup.select(\"ul.list1 a[href], a[href*='SampleSets']\")" } ], "dictionary_entries": [ { "line": 1674, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'List Samplesets'}" }, { "line": 1675, "key": "item", "value_expression": "{'title': title, 'provider_label': 'PCorgan', 'description_excerpt': f'PCorgan public Hauptwerk sample-set registry entry: {title}.', 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'registry_link', 'availability_status_hint': 'registry_entry'}" }, { "line": 1684, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'registry_entry', 'is_odf_only': False, 'requires_sample_set': False}" }, { "line": 1690, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1691, "key": "extensions", "value_expression": "{'source_fragment_html': str(link)}" }, { "line": 1674, "key": "position", "value_expression": "position" }, { "line": 1674, "key": "section_label", "value_expression": "'List Samplesets'" }, { "line": 1676, "key": "title", "value_expression": "title" }, { "line": 1677, "key": "provider_label", "value_expression": "'PCorgan'" }, { "line": 1678, "key": "description_excerpt", "value_expression": "f'PCorgan public Hauptwerk sample-set registry entry: {title}.'" }, { "line": 1679, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1680, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1681, "key": "access_model_hint", "value_expression": "'registry_link'" }, { "line": 1682, "key": "availability_status_hint", "value_expression": "'registry_entry'" }, { "line": 1685, "key": "package_title", "value_expression": "title" }, { "line": 1686, "key": "package_kind", "value_expression": "'registry_entry'" }, { "line": 1687, "key": "is_odf_only", "value_expression": "False" }, { "line": 1688, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1691, "key": "source_fragment_html", "value_expression": "str(link)" }, { "line": 1690, "key": "url", "value_expression": "detail_url" }, { "line": 1690, "key": "link_text", "value_expression": "title" }, { "line": 1690, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1686, "key": "package_kind", "value_expression": "'registry_entry'" }, { "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1688, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1687, "key": "is_odf_only", "value_expression": "False" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "line_start": 1683, "line_end": 1691, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"registry_entry\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "line_start": 1685, "line_end": 1693, "text": " \"package_title\": title,\n \"package_kind\": \"registry_entry\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)},\n },\n primary_id=slugify(title)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PcorganHauptwerkSamplesetsCrawler.parse_html", "line_start": 1684, "line_end": 1692, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"registry_entry\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)},\n }," } ] } }, { "catalogue_id": "S041", "source_key": "piotr_grabowski_go_versions", "name": "Piotr Grabowski GrandOrgue Versions", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Piotr Grabowski", "scope_description": "GrandOrgue versions of the producer's sample sets", "documentary_unit": "GrandOrgue conversion or compatibility record", "source_description": "The page documents installation, patching and feature differences between GrandOrgue and Hauptwerk versions. It states that some features are absent and that presentation videos were recorded using Hauptwerk.", "source_description_references": [ "piotr_grabowski_go_versions" ], "interpretation": "Shared recordings do not imply identical software behaviour. Link a GrandOrgue version to the corresponding product while preserving format, release and known limitations. This key and the instruments catalogue share a producer and much underlying material.", "rights_observation": "Software conversion information does not establish permission to redistribute the underlying recordings.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "piotr_grabowski_go_versions" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects article, list, paragraph and table nodes, with download-link fallback. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is GrandOrgue versions and their linked assets. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A port or format version should remain linked to its underlying product without collapsing edition and format into one identity. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "PiotrGrabowskiGoVersionsCrawler", "class_line": 881, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "PiotrGrabowskiGoVersionsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "piotr_grabowski_go_versions", "path": "aggregator/grandorgue_library_crawler.py", "line": 882, "class": "PiotrGrabowskiGoVersionsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_downloads", "path": "aggregator/grandorgue_library_crawler.py", "line": 884, "class": "PiotrGrabowskiGoVersionsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 886, "line_end": 959, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 910, "receiver": "link", "key": "is_odf_candidate", "expression": "link.get('is_odf_candidate')" }, { "line": 910, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 891, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 900, "receiver": "node", "key": "href", "expression": "node.get('href')" } ], "html_queries": [ { "line": 893, "expression": "soup.select('article, li, p, tr')" }, { "line": 890, "expression": "soup.select('a[href*=\"/downloads/\"]')" } ], "dictionary_entries": [ { "line": 900, "key": "url", "value_expression": "urljoin(source_url, node.get('href') or '')" }, { "line": 901, "key": "link_text", "value_expression": "normalize_title(node.get_text(' ', strip=True))" }, { "line": 902, "key": "position", "value_expression": "1" }, { "line": 903, "key": "is_download_candidate", "value_expression": "True" }, { "line": 904, "key": "is_odf_candidate", "value_expression": "True" }, { "line": 922, "key": "catalog", "value_expression": "{'position': position}" }, { "line": 923, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Piotr Grabowski', 'description_excerpt': _excerpt(text), 'access_model_hint': 'free_or_product_dependency', 'availability_status_hint': 'odf_available'}" }, { "line": 930, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'odf', 'package_format': _package_format(download_url) or 'zip', 'download_url': download_url, 'file_name': _file_name(download_url), 'requires_sample_set': True, 'requires_commercial_sample_set': any((term in lower for term in ('buy', 'paid', 'commercial'))), 'is_odf_only': True}" }, { "line": 940, "key": "compatibility", "value_expression": "[{'software_label': 'GrandOrgue', 'software_version': _first_match(text, '(?i)GrandOrgue\\\\s*([0-9][0-9A-Za-z._-]*)') or '', 'compatibility_status': 'supported_with_notes' if any((term in lower for term in ('note', 'limitation', 'warning'))) else 'supported', 'notes': text}]" }, { "line": 948, "key": "links", "value_expression": "links" }, { "line": 949, "key": "extensions", "value_expression": "{'source_fragment_html': str(parent)}" }, { "line": 922, "key": "position", "value_expression": "position" }, { "line": 924, "key": "title", "value_expression": "title" }, { "line": 925, "key": "producer_label", "value_expression": "'Piotr Grabowski'" }, { "line": 926, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 927, "key": "access_model_hint", "value_expression": "'free_or_product_dependency'" }, { "line": 928, "key": "availability_status_hint", "value_expression": "'odf_available'" }, { "line": 931, "key": "package_title", "value_expression": "title" }, { "line": 932, "key": "package_kind", "value_expression": "'odf'" }, { "line": 933, "key": "package_format", "value_expression": "_package_format(download_url) or 'zip'" }, { "line": 934, "key": "download_url", "value_expression": "download_url" }, { "line": 935, "key": "file_name", "value_expression": "_file_name(download_url)" }, { "line": 936, "key": "requires_sample_set", "value_expression": "True" }, { "line": 937, "key": "requires_commercial_sample_set", "value_expression": "any((term in lower for term in ('buy', 'paid', 'commercial')))" }, { "line": 938, "key": "is_odf_only", "value_expression": "True" }, { "line": 949, "key": "source_fragment_html", "value_expression": "str(parent)" }, { "line": 942, "key": "software_label", "value_expression": "'GrandOrgue'" }, { "line": 943, "key": "software_version", "value_expression": "_first_match(text, '(?i)GrandOrgue\\\\s*([0-9][0-9A-Za-z._-]*)') or ''" }, { "line": 944, "key": "compatibility_status", "value_expression": "'supported_with_notes' if any((term in lower for term in ('note', 'limitation', 'warning'))) else 'supported'" }, { "line": 945, "key": "notes", "value_expression": "text" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 932, "key": "package_kind", "value_expression": "'odf'" }, { "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 936, "key": "requires_sample_set", "value_expression": "True" }, { "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 938, "key": "is_odf_only", "value_expression": "True" }, { "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 925, "key": "producer_label", "value_expression": "'Piotr Grabowski'" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "line_start": 929, "line_end": 937, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"odf\",\n \"package_format\": _package_format(download_url) or \"zip\",\n \"download_url\": download_url,\n \"file_name\": _file_name(download_url),\n \"requires_sample_set\": True,\n \"requires_commercial_sample_set\": any(term in lower for term in (\"buy\", \"paid\", \"commercial\"))," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "line_start": 933, "line_end": 941, "text": " \"package_format\": _package_format(download_url) or \"zip\",\n \"download_url\": download_url,\n \"file_name\": _file_name(download_url),\n \"requires_sample_set\": True,\n \"requires_commercial_sample_set\": any(term in lower for term in (\"buy\", \"paid\", \"commercial\")),\n \"is_odf_only\": True,\n },\n \"compatibility\": [\n {" }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "line_start": 935, "line_end": 943, "text": " \"file_name\": _file_name(download_url),\n \"requires_sample_set\": True,\n \"requires_commercial_sample_set\": any(term in lower for term in (\"buy\", \"paid\", \"commercial\")),\n \"is_odf_only\": True,\n },\n \"compatibility\": [\n {\n \"software_label\": \"GrandOrgue\",\n \"software_version\": _first_match(text, r\"(?i)GrandOrgue\\s*([0-9][0-9A-Za-z._-]*)\") or \"\"," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiGoVersionsCrawler.parse_html", "line_start": 922, "line_end": 930, "text": " \"catalog\": {\"position\": position},\n \"item\": {\n \"title\": title,\n \"producer_label\": \"Piotr Grabowski\",\n \"description_excerpt\": _excerpt(text),\n \"access_model_hint\": \"free_or_product_dependency\",\n \"availability_status_hint\": \"odf_available\",\n },\n \"package\": {" } ] } }, { "catalogue_id": "S042", "source_key": "piotr_grabowski_instruments", "name": "Piotr Grabowski Hauptwerk Instruments", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Piotr Grabowski", "scope_description": "International product catalogue with paid, free and unreleased entries", "documentary_unit": "Instrument-based sample-set product", "source_description": "Catalogue cards expose builder, date, stops and keyboard counts. Free products and unreleased entries are marked separately. The list also contains a harmonium, so the producer's instrument catalogue is broader than pipe organs alone.", "source_description_references": [ "piotr_grabowski_instruments" ], "interpretation": "Preserve release status and instrument type. The card's construction date belongs to the represented instrument, not automatically to the sample set. Duplicate discovery through other registries should resolve to a product record rather than another physical organ.", "rights_observation": "The producer asserts copyright; free products still require their own licence check.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "piotr_grabowski_instruments" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects links to instrument pages on the producer host. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is Hauptwerk instrument products. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "The same represented organ can have several products or format versions; catalogue identity must retain that distinction. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "PiotrGrabowskiInstrumentsCrawler", "class_line": 1266, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "PiotrGrabowskiInstrumentsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "piotr_grabowski_instruments", "path": "aggregator/grandorgue_library_crawler.py", "line": 1267, "class": "PiotrGrabowskiInstrumentsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 1269, "class": "PiotrGrabowskiInstrumentsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1271, "line_end": 1329, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1276, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 1275, "expression": "soup.select('a[href*=\"piotrgrabowski.pl/\"]')" } ], "dictionary_entries": [ { "line": 1292, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Instruments'}" }, { "line": 1293, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Piotr Grabowski', 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'free_download' if is_free else 'commercial_or_donationware', 'availability_status_hint': 'producer_catalog_entry'}" }, { "line": 1302, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'sample_set', 'is_odf_only': False, 'requires_sample_set': False, 'is_demo': False}" }, { "line": 1309, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': title, 'builder_label': builder, 'year_label': year, 'manual_count': _first_int(keyboards), 'pedal_present': 'p' in keyboards.lower(), 'stop_count': stops}" }, { "line": 1318, "key": "links", "value_expression": "[{'url': detail_url, 'link_text': title, 'is_product_homepage_candidate': True}]" }, { "line": 1319, "key": "extensions", "value_expression": "{'source_fragment_html': str(link)}" }, { "line": 1292, "key": "position", "value_expression": "position" }, { "line": 1292, "key": "section_label", "value_expression": "'Instruments'" }, { "line": 1294, "key": "title", "value_expression": "title" }, { "line": 1295, "key": "producer_label", "value_expression": "'Piotr Grabowski'" }, { "line": 1296, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1297, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1298, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1299, "key": "access_model_hint", "value_expression": "'free_download' if is_free else 'commercial_or_donationware'" }, { "line": 1300, "key": "availability_status_hint", "value_expression": "'producer_catalog_entry'" }, { "line": 1303, "key": "package_title", "value_expression": "title" }, { "line": 1304, "key": "package_kind", "value_expression": "'sample_set'" }, { "line": 1305, "key": "is_odf_only", "value_expression": "False" }, { "line": 1306, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1307, "key": "is_demo", "value_expression": "False" }, { "line": 1310, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 1311, "key": "represented_organ_label", "value_expression": "title" }, { "line": 1312, "key": "builder_label", "value_expression": "builder" }, { "line": 1313, "key": "year_label", "value_expression": "year" }, { "line": 1314, "key": "manual_count", "value_expression": "_first_int(keyboards)" }, { "line": 1315, "key": "pedal_present", "value_expression": "'p' in keyboards.lower()" }, { "line": 1316, "key": "stop_count", "value_expression": "stops" }, { "line": 1319, "key": "source_fragment_html", "value_expression": "str(link)" }, { "line": 1318, "key": "url", "value_expression": "detail_url" }, { "line": 1318, "key": "link_text", "value_expression": "title" }, { "line": 1318, "key": "is_product_homepage_candidate", "value_expression": "True" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1316, "key": "stop_count", "value_expression": "stops" }, { "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1314, "key": "manual_count", "value_expression": "_first_int(keyboards)" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1304, "key": "package_kind", "value_expression": "'sample_set'" }, { "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1306, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1305, "key": "is_odf_only", "value_expression": "False" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "line_start": 1313, "line_end": 1321, "text": " \"year_label\": year,\n \"manual_count\": _first_int(keyboards),\n \"pedal_present\": \"p\" in keyboards.lower(),\n \"stop_count\": stops,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)},\n },\n primary_id=slugify(path)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "line_start": 1311, "line_end": 1319, "text": " \"represented_organ_label\": title,\n \"builder_label\": builder,\n \"year_label\": year,\n \"manual_count\": _first_int(keyboards),\n \"pedal_present\": \"p\" in keyboards.lower(),\n \"stop_count\": stops,\n },\n \"links\": [{\"url\": detail_url, \"link_text\": title, \"is_product_homepage_candidate\": True}],\n \"extensions\": {\"source_fragment_html\": str(link)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "line_start": 1301, "line_end": 1309, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": False,\n },\n \"organ_relation\": {" }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "line_start": 1303, "line_end": 1311, "text": " \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "PiotrGrabowskiInstrumentsCrawler.parse_html", "line_start": 1302, "line_end": 1310, "text": " \"package\": {\n \"package_title\": title,\n \"package_kind\": \"sample_set\",\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": False,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\"," } ] } }, { "catalogue_id": "S043", "source_key": "pipedreams", "name": "Pipedreams Organ Gallery", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Pipedreams", "scope_description": "Instruments documented in a broadcasting context", "documentary_unit": "Organ-gallery profile linked to programme and performance material", "source_description": "The live site is organised around programmes and performances, with dated episodes and related material. Aggregator selects organ-gallery information from that broader broadcasting resource.", "source_description_references": [ "pipedreams", "selection_scope", "record_pipedreams" ], "interpretation": "The recording, programme, performer and instrument are different entities. A programme's publication date cannot be assigned as a date for an organ's construction or documented specification. The inspected homepage does not establish an exhaustive inventory of organs.", "rights_observation": "Streaming or linking to a programme does not establish permission to redistribute its audio.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "pipedreams", "selection_scope", "record_pipedreams" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_pipedreams", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Organ-gallery hub connecting an instrument to broadcast programmes and external documents", "repeated_information": "Related-document links followed by many programme cards.", "date_and_state_distinctions": "The instrument construction year in the title differs from broadcast episode dates.", "interpretive_limit": "A gallery page primarily aggregates performance context. Linked specifications were not downloaded.", "observations": [ "Concert venue, city and country are in the title.", "Manufacturer and construction year are in the title.", "A manufacturer PDF and an external organ account are linked.", "Venue, manufacturer and organ-description links are present." ], "uninspected_or_unfilled_material": [ "Specification is linked as an external manufacturer PDF, not transcribed in the inspected page.", "Programme cards establish a route to broadcasts; audio files were not inspected." ], "narrative": "The Calgary gallery page identifies an instrument and links it to programme cards, a manufacturer document and another organ account. The external specification and audio were not inspected. The page is useful as a route to performance and documentary material, but its programme dates do not date the organ specification." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler interprets embedded application data and profile structures, with separate discovery logic for gallery pages.", "transformation": "Profile title and rich body supply identification and description. Image resources, related links and audio contribution results have specialised handlers. Publication date, profile ID, resource type and canonical slug are retained. The transform intentionally leaves the raw specification empty and adds a source-scope note.", "limitations": "That empty specification is an output decision, not an audit of every gallery page. Publication date concerns the profile, not organ construction. A title-derived builder/year is less explicit than a labelled source field; media and broadcast associations should not be treated as organ measurements." }, "implementation": { "class": "PipedreamsCrawler", "class_line": 2961, "path": "aggregator/static_html_crawler.py", "inheritance": [ "PipedreamsCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "pipedreams", "path": "aggregator/static_html_crawler.py", "line": 2962, "class": "PipedreamsCrawler" }, "id_column": { "value": "pipedreams_id", "path": "aggregator/static_html_crawler.py", "line": 2963, "class": "PipedreamsCrawler" }, "default_country": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 28, "class": "StaticHtmlOrganCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://www.pipedreams.org/page/organ-gallery" ], "path": "aggregator/static_html_crawler.py", "line": 2966, "class": "PipedreamsCrawler" }, "detail_url_patterns": { "value": [ "/profile/" ], "path": "aggregator/static_html_crawler.py", "line": 2967, "class": "PipedreamsCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://www.pipedreams.org", "path": "aggregator/static_html_crawler.py", "line": 2965, "class": "PipedreamsCrawler" }, "COUNTRY_SEGMENTS": { "value": { "australia": "Australia", "austria": "Austria", "belgium": "Belgium", "canada": "Canada", "china": "China", "czech-republic": "Czech Republic", "denmark": "Denmark", "finland": "Finland", "france": "France", "germany": "Germany", "hungary": "Hungary", "iceland": "Iceland", "italy": "Italy", "japan": "Japan", "latvia": "Latvia", "luxembourg": "Luxembourg", "mexico": "Mexico", "new-zealand": "New Zealand", "norway": "Norway", "russia": "Russia", "south-africa": "South Africa", "spain": "Spain", "sweden": "Sweden", "switzerland": "Switzerland", "the-netherlands": "Netherlands", "united-kingdom": "United Kingdom" }, "path": "aggregator/static_html_crawler.py", "line": 2969, "class": "PipedreamsCrawler" } }, "methods": [ { "symbol": "PipedreamsCrawler._append_reference", "path": "aggregator/static_html_crawler.py", "line_start": 3058, "line_end": 3066, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_save_reference" ], "literal_get_calls": [ { "line": 3059, "receiver": "ref", "key": "url", "expression": "ref.get('url', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipedreamsCrawler._location_from_profile_url", "path": "aggregator/static_html_crawler.py", "line_start": 3276, "line_end": 3283, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_title_case_segment" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipedreamsCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 3068, "line_end": 3094, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_canonical_site_url", "_clean_text", "_country_region_from_url", "_dedupe_refs", "_identifier_from_url", "_is_detail_url", "_location_from_profile_url", "_next_data", "_page_payload", "_reference_from_parts", "_rich_body", "_rich_links", "_summary_from_link_text", "_title_from_url" ], "literal_get_calls": [ { "line": 3072, "receiver": "page", "key": "title", "expression": "page.get('title')" } ], "html_queries": [], "dictionary_entries": [ { "line": 3090, "key": "region_page", "value_expression": "page_url" }, { "line": 3090, "key": "region_title", "value_expression": "region_title" } ], "subscript_assignments": [] }, { "symbol": "PipedreamsCrawler._parse_profile_title", "path": "aggregator/static_html_crawler.py", "line_start": 3245, "line_end": 3259, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipedreamsCrawler._title_case_segment", "path": "aggregator/static_html_crawler.py", "line_start": 3285, "line_end": 3286, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipedreamsCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 2998, "line_end": 3056, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_append_reference", "_fetch_html", "_optional_int", "_parse_index_page", "_region_page_urls", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 2999, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 3000, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 3002, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 3001, "receiver": "kwargs", "key": "max_region_pages", "expression": "kwargs.get('max_region_pages')" }, { "line": 3001, "receiver": "self.config", "key": "max_region_pages", "expression": "self.config.get('max_region_pages')" }, { "line": 3044, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 3096, "line_end": 3162, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_audio_from_contribution_results", "_body_text", "_clean_text", "_country_region_from_url", "_images_from_profile", "_next_data", "_page_payload", "_parse_profile_title", "_related_links", "_rich_body" ], "literal_get_calls": [ { "line": 3097, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 3097, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 3104, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 3105, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 3106, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 3107, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 3110, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 3111, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 3101, "receiver": "profile", "key": "title", "expression": "profile.get('title')" }, { "line": 3101, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 3103, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 3103, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 3112, "receiver": "profile", "key": "descriptionText", "expression": "profile.get('descriptionText')" }, { "line": 3142, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 3159, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 3144, "receiver": "profile", "key": "canonicalSlug", "expression": "profile.get('canonicalSlug')" }, { "line": 3148, "receiver": "profile", "key": "id", "expression": "profile.get('id')" }, { "line": 3149, "receiver": "profile", "key": "resourceType", "expression": "profile.get('resourceType')" }, { "line": 3150, "receiver": "profile", "key": "publishDate", "expression": "profile.get('publishDate')" }, { "line": 3158, "receiver": "raw_data", "key": "pipedreams_id", "expression": "raw_data.get('pipedreams_id')" }, { "line": 3158, "receiver": "profile", "key": "canonicalSlug", "expression": "profile.get('canonicalSlug')" }, { "line": 3139, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 3151, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" } ], "html_queries": [], "dictionary_entries": [ { "line": 3118, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': country, 'region': region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 3128, "key": "builder_history", "value_expression": "[{'builder_name': builder, 'year': year, 'activity': 'builder'}] if builder else []" }, { "line": 3129, "key": "specifications", "value_expression": "{'raw_specification': '', 'notes': [{'type': 'source_scope', 'text': 'Pipedreams profiles primarily provide gallery/media context; stoplists are not exposed consistently.'}]}" }, { "line": 3138, "key": "media", "value_expression": "{'images': images, 'audio': audio}" }, { "line": 3139, "key": "links", "value_expression": "[{'link': raw_data.get('url'), 'type': 'source'}] + related_links" }, { "line": 3140, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': description}] if description else []" }, { "line": 3141, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', ''), 'profile_title': title, 'canonical_slug': profile.get('canonicalSlug') or ''}" }, { "line": 3146, "key": "extensions", "value_expression": "{self.source_key: {'profile_id': profile.get('id'), 'resource_type': profile.get('resourceType'), 'publish_date': profile.get('publishDate'), 'region_title': raw_data.get('region') or ''}}" }, { "line": 3130, "key": "raw_specification", "value_expression": "''" }, { "line": 3131, "key": "notes", "value_expression": "[{'type': 'source_scope', 'text': 'Pipedreams profiles primarily provide gallery/media context; stoplists are not exposed consistently.'}]" }, { "line": 3138, "key": "images", "value_expression": "images" }, { "line": 3138, "key": "audio", "value_expression": "audio" }, { "line": 3142, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 3143, "key": "profile_title", "value_expression": "title" }, { "line": 3144, "key": "canonical_slug", "value_expression": "profile.get('canonicalSlug') or ''" }, { "line": 3120, "key": "status", "value_expression": "'current_location'" }, { "line": 3121, "key": "country", "value_expression": "country" }, { "line": 3122, "key": "region", "value_expression": "region" }, { "line": 3123, "key": "town", "value_expression": "town" }, { "line": 3124, "key": "building", "value_expression": "building" }, { "line": 3125, "key": "raw_str", "value_expression": "title" }, { "line": 3148, "key": "profile_id", "value_expression": "profile.get('id')" }, { "line": 3149, "key": "resource_type", "value_expression": "profile.get('resourceType')" }, { "line": 3150, "key": "publish_date", "value_expression": "profile.get('publishDate')" }, { "line": 3151, "key": "region_title", "value_expression": "raw_data.get('region') or ''" }, { "line": 3128, "key": "builder_name", "value_expression": "builder" }, { "line": 3128, "key": "year", "value_expression": "year" }, { "line": 3128, "key": "activity", "value_expression": "'builder'" }, { "line": 3133, "key": "type", "value_expression": "'source_scope'" }, { "line": 3134, "key": "text", "value_expression": "'Pipedreams profiles primarily provide gallery/media context; stoplists are not exposed consistently.'" }, { "line": 3139, "key": "link", "value_expression": "raw_data.get('url')" }, { "line": 3139, "key": "type", "value_expression": "'source'" }, { "line": 3140, "key": "type", "value_expression": "'source_text'" }, { "line": 3140, "key": "text", "value_expression": "description" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3130, "key": "raw_specification", "value_expression": "''" }, { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3121, "key": "country", "value_expression": "country" }, { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 3125, "key": "raw_str", "value_expression": "title" }, { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "html", "value_expression": "raw_data.get('html')", "line": 3097, "kind": "input read" }, { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "raw_html", "value_expression": "raw_data.get('raw_html')", "line": 3097, "kind": "input read" }, { "symbol": "PipedreamsCrawler.transform", "path": "aggregator/static_html_crawler.py", "key": "town", "value_expression": "raw_data.get('town')", "line": 3104, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "PipedreamsCrawler.transform", "line_start": 3127, "line_end": 3135, "text": " ],\n \"builder_history\": [{\"builder_name\": builder, \"year\": year, \"activity\": \"builder\"}] if builder else [],\n \"specifications\": {\n \"raw_specification\": \"\",\n \"notes\": [\n {\n \"type\": \"source_scope\",\n \"text\": \"Pipedreams profiles primarily provide gallery/media context; stoplists are not exposed consistently.\",\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "PipedreamsCrawler.transform", "line_start": 3118, "line_end": 3126, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": country,\n \"region\": region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "PipedreamsCrawler.transform", "line_start": 3122, "line_end": 3130, "text": " \"region\": region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }\n ],\n \"builder_history\": [{\"builder_name\": builder, \"year\": year, \"activity\": \"builder\"}] if builder else [],\n \"specifications\": {\n \"raw_specification\": \"\"," }, { "path": "aggregator/static_html_crawler.py", "symbol": "PipedreamsCrawler.transform", "line_start": 3096, "line_end": 3102, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n html = raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\"\n next_data = self._next_data(html)\n profile = self._page_payload(next_data, \"profile\")\n body = self._rich_body(profile)\n title = self._clean_text(profile.get(\"title\") or raw_data.get(\"title\") or \"\")\n year, builder, building, town, region_from_title = self._parse_profile_title(title)" }, { "path": "aggregator/static_html_crawler.py", "symbol": "PipedreamsCrawler.transform", "line_start": 3101, "line_end": 3109, "text": " title = self._clean_text(profile.get(\"title\") or raw_data.get(\"title\") or \"\")\n year, builder, building, town, region_from_title = self._parse_profile_title(title)\n country_from_url, region_from_url = self._country_region_from_url(raw_data.get(\"url\") or \"\", raw_data.get(\"region\") or \"\")\n town = raw_data.get(\"town\") or town\n building = raw_data.get(\"building\") or building or title\n region = raw_data.get(\"region\") or region_from_title or region_from_url\n country = raw_data.get(\"country\") or country_from_url\n if region and country and region.lower() == country.lower():\n region = \"\"" } ] } }, { "catalogue_id": "S044", "source_key": "pipeorganmap", "name": "Pipe Organ Map", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Leif-Erik Hallmann / Pipe Organ Map", "scope_description": "International map and database", "documentary_unit": "Organ record linked to builders and sample-set creators", "source_description": "The site combines map-based discovery with organ, builder and sample-set-creator records and links to API documentation. Its footer identifies Leif-Erik Hallmann and labels content CC BY-SA 4.0.", "source_description_references": [ "pipeorganmap", "record_pipeorganmap" ], "interpretation": "The explicit separation of builders and sample-set creators supports cross-domain links. Map coordinates and popular-organ selections should not be treated as measures of historical coverage. Any claim imported from another source still needs its attribution.", "rights_observation": "The footer states CC BY-SA 4.0 for content; third-party material requires its own credit and scope check.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "pipeorganmap", "record_pipeorganmap" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_pipeorganmap", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Linked organ profile with metadata, division stop table, map and source links", "repeated_information": "Stop divisions, nearby instruments, same-builder instruments and similarity suggestions.", "date_and_state_distinctions": "Build year and narrative interventions coexist with a version-history link.", "interpretive_limit": "Sample-set suggestions are recommendation links, not proof of sampled-instrument identity. Only the pipe-organ record is included here.", "observations": [ "Street address, city and country are supplied.", "Map and distance-based neighbouring instruments are displayed.", "Builder is a linked entity.", "Dated narrative distinguishes installation and later relocations.", "Named divisions contain stop names and pitches.", "Division count and manual/pedal grouping are supplied.", "Key action is a labelled attribute; prose also describes stop action.", "Tuning is a labelled attribute.", "Case and console arrangement are described in prose.", "The record cites Musique et musiciens and INdO.", "Related instrument, builder and upstream record links are explicit.", "A version-history route is exposed." ], "uninspected_or_unfilled_material": [ "The selected record explicitly states that no video/audio samples are available." ], "narrative": "The selected Lille record combines technical attributes, a division stoplist, narrative, map and explicit upstream source links. Nearby and similar instruments are offered as discovery routes. Sample-set recommendations do not establish that a product was recorded from this organ. The page explicitly reported no available audio/video samples for this record; no source-wide absence follows." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The subclass extracts organ links from the map/index representation and provides an additional specification-selection routine.", "transformation": "It uses the shared HTML transform after source-specific discovery and stoplist handling. Map/list context, textual description, division strings and images are different forms of evidence. Source identifiers derived from URLs identify documentary entries rather than proving unique physical instruments.", "limitations": "A map entry and a coordinate measurement are not interchangeable. The generic transform can retain text without resolving all fields, and its first-year fallback can be too broad for a narrative containing several dates. Geographic reach must be established from a dated collection inventory." }, "implementation": { "class": "PipeOrganMapCrawler", "class_line": 1274, "path": "aggregator/static_html_crawler.py", "inheritance": [ "PipeOrganMapCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "pipeorganmap", "path": "aggregator/static_html_crawler.py", "line": 1275, "class": "PipeOrganMapCrawler" }, "id_column": { "value": "pipeorganmap_id", "path": "aggregator/static_html_crawler.py", "line": 1276, "class": "PipeOrganMapCrawler" }, "default_country": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 28, "class": "StaticHtmlOrganCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [ "https://pipeorganmap.com/organs" ], "path": "aggregator/static_html_crawler.py", "line": 1279, "class": "PipeOrganMapCrawler" }, "detail_url_patterns": { "value": [ "/organ/" ], "path": "aggregator/static_html_crawler.py", "line": 1280, "class": "PipeOrganMapCrawler" }, "exclude_url_patterns": { "value": [ "/preview", ".pdf", "/liked", "/nearby" ], "path": "aggregator/static_html_crawler.py", "line": 1281, "class": "PipeOrganMapCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://pipeorganmap.com", "path": "aggregator/static_html_crawler.py", "line": 1278, "class": "PipeOrganMapCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipeOrganMapCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 1320, "line_end": 1336, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 1321, "expression": "soup.find_all('table')" }, { "line": 1325, "expression": "stoplist.find_all('tr')" }, { "line": 1331, "expression": "row.find_all(['th', 'td'], recursive=False)" }, { "line": 1328, "expression": "rows[0].find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "PipeOrganMapCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 1283, "line_end": 1300, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_summary_from_card", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 1286, "expression": "soup.select(\"a[href^='/organ/'], a[href*='pipeorganmap.com/organ/']\")" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 78, "line_end": 129, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_html", "_identifier_from_url", "_optional_int", "_parse_index_page", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 79, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 80, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 81, "receiver": "kwargs", "key": "start_urls", "expression": "kwargs.get('start_urls')" }, { "line": 117, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 434, "line_end": 516, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_build_involved", "_extract_images", "_extract_links", "_extract_metadata", "_extract_raw_spec", "_extract_table_rows", "_extract_title", "_first_int", "_first_metadata", "_first_year", "_location_parts", "_main_text", "_safe_json_loads" ], "literal_get_calls": [ { "line": 435, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 440, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder')" }, { "line": 441, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year')" }, { "line": 442, "receiver": "raw_data", "key": "manual_count", "expression": "raw_data.get('manual_count')" }, { "line": 443, "receiver": "raw_data", "key": "stop_count", "expression": "raw_data.get('stop_count')" }, { "line": 444, "receiver": "raw_data", "key": "rank_count", "expression": "raw_data.get('rank_count')" }, { "line": 513, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 442, "receiver": "metadata", "key": "manuals", "expression": "metadata.get('manuals')" }, { "line": 442, "receiver": "metadata", "key": "manual", "expression": "metadata.get('manual')" }, { "line": 443, "receiver": "metadata", "key": "stops", "expression": "metadata.get('stops')" }, { "line": 443, "receiver": "metadata", "key": "stop", "expression": "metadata.get('stop')" }, { "line": 444, "receiver": "metadata", "key": "ranks", "expression": "metadata.get('ranks')" }, { "line": 444, "receiver": "metadata", "key": "rank", "expression": "metadata.get('rank')" }, { "line": 480, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 453, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 454, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 479, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 487, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "locations", "value_expression": "[{'status': 'current_location', 'country': raw_data.get('country') or self.default_country, 'region': raw_data.get('region') or self.default_region, 'town': town, 'building': building, 'raw_str': title}]" }, { "line": 460, "key": "involved", "value_expression": "self._build_involved(builder, year)" }, { "line": 461, "key": "events", "value_expression": "[{'type': 'construction', 'date': str(year), 'description': builder}] if year else []" }, { "line": 462, "key": "specifications", "value_expression": "{'manual_count': manual_count, 'stop_count': stop_count, 'rank_count': rank_count, 'stops_by_division': stops_by_division, 'stops': [{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops], 'raw_spec': raw_spec, 'technical_rows_unstructured': [{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows}" }, { "line": 479, "key": "media", "value_expression": "{'images': self._extract_images(soup, raw_data.get('url') or '')}" }, { "line": 480, "key": "links", "value_expression": "self._extract_links(soup, raw_data.get('url') or '')" }, { "line": 481, "key": "descriptions", "value_expression": "[{'type': 'source_text', 'text': text}] if text else []" }, { "line": 482, "key": "other_identifiers", "value_expression": "[{'type': self.id_column, 'value': raw_data.get(self.id_column)}]" }, { "line": 483, "key": "extensions", "value_expression": "{self.source_key: {'title': title, 'metadata': metadata, 'raw_data': self._safe_json_loads(raw_data.get('raw_data_json'), {}), 'reference_context': {key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}}}" }, { "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 467, "key": "stops", "value_expression": "[{'name': stop, 'division': division, 'raw_str': stop} for division, stops in stops_by_division.items() for stop in stops]" }, { "line": 472, "key": "raw_spec", "value_expression": "raw_spec" }, { "line": 473, "key": "technical_rows_unstructured", "value_expression": "[{'label': key, 'value': value} for key, value in metadata.items() if value and key not in {'builder', 'year', 'manuals', 'manual', 'stops', 'stop', 'ranks', 'rank'}] + table_rows" }, { "line": 479, "key": "images", "value_expression": "self._extract_images(soup, raw_data.get('url') or '')" }, { "line": 452, "key": "status", "value_expression": "'current_location'" }, { "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "line": 454, "key": "region", "value_expression": "raw_data.get('region') or self.default_region" }, { "line": 455, "key": "town", "value_expression": "town" }, { "line": 456, "key": "building", "value_expression": "building" }, { "line": 457, "key": "raw_str", "value_expression": "title" }, { "line": 482, "key": "type", "value_expression": "self.id_column" }, { "line": 482, "key": "value", "value_expression": "raw_data.get(self.id_column)" }, { "line": 485, "key": "title", "value_expression": "title" }, { "line": 486, "key": "metadata", "value_expression": "metadata" }, { "line": 487, "key": "raw_data", "value_expression": "self._safe_json_loads(raw_data.get('raw_data_json'), {})" }, { "line": 488, "key": "reference_context", "value_expression": "{key: raw_data.get(key) for key in (self.id_column, 'title', 'country', 'region', 'town', 'building', 'builder', 'year', 'manual_count', 'stop_count', 'rank_count', 'url') if raw_data.get(key) not in (None, '')}" }, { "line": 461, "key": "type", "value_expression": "'construction'" }, { "line": 461, "key": "date", "value_expression": "str(year)" }, { "line": 461, "key": "description", "value_expression": "builder" }, { "line": 468, "key": "name", "value_expression": "stop" }, { "line": 468, "key": "division", "value_expression": "division" }, { "line": 468, "key": "raw_str", "value_expression": "stop" }, { "line": 481, "key": "type", "value_expression": "'source_text'" }, { "line": 481, "key": "text", "value_expression": "text" }, { "line": 474, "key": "label", "value_expression": "key" }, { "line": 474, "key": "value", "value_expression": "value" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 464, "key": "stop_count", "value_expression": "stop_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 463, "key": "manual_count", "value_expression": "manual_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 465, "key": "rank_count", "value_expression": "rank_count" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 466, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 453, "key": "country", "value_expression": "raw_data.get('country') or self.default_country" }, { "symbol": "StaticHtmlOrganCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 457, "key": "raw_str", "value_expression": "title" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 461, "line_end": 469, "text": " \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 460, "line_end": 468, "text": " \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 462, "line_end": 470, "text": " \"specifications\": {\n \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 463, "line_end": 471, "text": " \"manual_count\": manual_count,\n \"stop_count\": stop_count,\n \"rank_count\": rank_count,\n \"stops_by_division\": stops_by_division,\n \"stops\": [\n {\"name\": stop, \"division\": division, \"raw_str\": stop}\n for division, stops in stops_by_division.items()\n for stop in stops\n ]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 450, "line_end": 458, "text": " \"locations\": [\n {\n \"status\": \"current_location\",\n \"country\": raw_data.get(\"country\") or self.default_country,\n \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }" }, { "path": "aggregator/static_html_crawler.py", "symbol": "StaticHtmlOrganCrawler.transform", "line_start": 454, "line_end": 462, "text": " \"region\": raw_data.get(\"region\") or self.default_region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": title,\n }\n ],\n \"involved\": self._build_involved(builder, year),\n \"events\": [{\"type\": \"construction\", \"date\": str(year), \"description\": builder}] if year else [],\n \"specifications\": {" } ] } }, { "catalogue_id": "S045", "source_key": "ridim_iconography", "name": "RIdIM Music Iconography", "selection_role": "registered_source", "parent_source_key": null, "attribution": "RIdIM Germany / Bayerische Staatsbibliothek", "scope_description": "Visual representations of music and instruments", "documentary_unit": "Music-iconography catalogue record", "source_description": "The configured endpoint is the German RIdIM catalogue on musiconn, with basic and advanced searching. Its interface credits the Bayerische Staatsbibliothek and attaches specific licences to displayed collection images.", "source_description_references": [ "ridim_iconography", "ridim_imprint" ], "interpretation": "A depicted organ is not a surviving instrument record. Object identifier, holding institution, creator, date and depicted subject must remain distinct. The endpoint should not be cited as though it covered every international RIdIM database.", "rights_observation": "Displayed images carry different licences. A licence beside one illustration does not cover the entire catalogue.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "ridim_iconography", "ridim_imprint" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Result and detail material is merged into an iconographic record; query and result context remain part of its acquisition history.", "transformation": "The normalizer separates artwork, holding institution, iconographic subject matter, media and links. Native record identifiers, local object numbers and sigla have distinct roles. Fallback identity can be derived from identifying text. Original detail-field labels remain relevant to interpreting the constructed artwork and iconography fields.", "limitations": "An image depicting an organ is evidence about a depiction, not automatically about a surviving instrument. Production date, depicted event date and catalogue modification date must be distinguished. An image reference does not establish redistribution rights or physical access to the object." }, "implementation": { "class": "RidimIconographyCrawler", "class_line": 27, "path": "aggregator/ridim_iconography_crawler.py", "inheritance": [ "RidimIconographyCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "ridim_id", "path": "aggregator/ridim_iconography_crawler.py", "line": 35, "class": "RidimIconographyCrawler" }, "record_label": { "value": "iconography records", "path": "aggregator/ridim_iconography_crawler.py", "line": 36, "class": "RidimIconographyCrawler" } }, "methods": [ { "symbol": "RidimIconographyCrawler._crawl_options", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 518, "line_end": 531, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [ { "line": 524, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 524, "receiver": "source", "key": "lookfor", "expression": "source.get('lookfor')" }, { "line": 524, "receiver": "self.config", "key": "default_query", "expression": "self.config.get('default_query')" }, { "line": 530, "receiver": "source", "key": "fetch_details", "expression": "source.get('fetch_details')" }, { "line": 527, "receiver": "source", "key": "type", "expression": "source.get('type')" }, { "line": 530, "receiver": "self.config", "key": "fetch_details_on_crawl", "expression": "self.config.get('fetch_details_on_crawl', True)" }, { "line": 529, "receiver": "source", "key": "max_pages", "expression": "source.get('max_pages')" }, { "line": 529, "receiver": "self.config", "key": "max_pages", "expression": "self.config.get('max_pages')" }, { "line": 528, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 528, "receiver": "self.config", "key": "rss_limit", "expression": "self.config.get('rss_limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 526, "key": "query", "value_expression": "query" }, { "line": 527, "key": "type", "value_expression": "_clean_text(source.get('type') or 'AllFields')" }, { "line": 528, "key": "limit", "value_expression": "min(50, max(1, _safe_int(source.get('limit') or self.config.get('rss_limit'), default=50)))" }, { "line": 529, "key": "max_pages", "value_expression": "max(1, _safe_int(source.get('max_pages') or self.config.get('max_pages'), default=1))" }, { "line": 530, "key": "fetch_details", "value_expression": "_read_bool(source.get('fetch_details'), bool(self.config.get('fetch_details_on_crawl', True)))" } ], "subscript_assignments": [ { "line": 523, "value_line": 523, "key": "query", "target_expression": "source['query']", "value_expression": "target" } ] }, { "symbol": "RidimIconographyCrawler._respectful_pause", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 544, "line_end": 555, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [ { "line": 546, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 5.0)" }, { "line": 552, "receiver": "self.config", "key": "rate_limit_jitter", "expression": "self.config.get('rate_limit_jitter', 0.2)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "RidimIconographyCrawler.crawl", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 94, "line_end": 156, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [ "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "fetch_detail", "is_scrape_stop_requested", "mark_scrape_stopped", "search_rss" ], "literal_get_calls": [ { "line": 96, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 122, "key": "lookfor", "value_expression": "options['query']" }, { "line": 122, "key": "type", "value_expression": "options['type']" } ], "subscript_assignments": [ { "line": 122, "value_line": 122, "key": "query", "target_expression": "record['query']", "value_expression": "{'lookfor': options['query'], 'type': options['type']}" }, { "line": 123, "value_line": 123, "key": "result_index", "target_expression": "record['result_index']", "value_expression": "result_index" }, { "line": 124, "value_line": 124, "key": "declared_total_records", "target_expression": "record['declared_total_records']", "value_expression": "declared_total or None" }, { "line": 125, "value_line": 125, "key": "search_result_count", "target_expression": "record['search_result_count']", "value_expression": "page_total or None" } ] }, { "symbol": "RidimIconographyCrawler.fetch_detail", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 211, "line_end": 217, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [ "parse_detail_html" ], "literal_get_calls": [ { "line": 214, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "RidimIconographyCrawler.normalized_batch_record", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 383, "line_end": 402, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 396, "value_line": 396, "key": "transform_run_id", "target_expression": "identity['transform_run_id']", "value_expression": "process_id" }, { "line": 397, "value_line": 397, "key": "artifact_ids", "target_expression": "identity['artifact_ids']", "value_expression": "list(artifact_ids or [])" }, { "line": 399, "value_line": 399, "key": "url", "target_expression": "record.setdefault('source_snapshot', {})['url']", "value_expression": "source_url" }, { "line": 401, "value_line": 401, "key": "retrieved_at", "target_expression": "record.setdefault('source', {})['retrieved_at']", "value_expression": "retrieved_at" } ] }, { "symbol": "RidimIconographyCrawler.parse_detail_html", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 219, "line_end": 282, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [ { "line": 223, "receiver": "canonical_tag", "key": "href", "expression": "canonical_tag.get('href')" }, { "line": 238, "receiver": "detail_fields", "key": "Title", "expression": "detail_fields.get('Title')" }, { "line": 255, "receiver": "detail_fields", "key": "Record ID", "expression": "detail_fields.get('Record ID')" }, { "line": 256, "receiver": "detail_fields", "key": "RIdIM siglum", "expression": "detail_fields.get('RIdIM siglum')" }, { "line": 258, "receiver": "detail_fields", "key": "Creator/Producer", "expression": "detail_fields.get('Creator/Producer')" }, { "line": 260, "receiver": "detail_fields", "key": "Production date", "expression": "detail_fields.get('Production date')" }, { "line": 261, "receiver": "detail_fields", "key": "Category", "expression": "detail_fields.get('Category')" }, { "line": 262, "receiver": "detail_fields", "key": "Object type", "expression": "detail_fields.get('Object type')" }, { "line": 265, "receiver": "detail_fields", "key": "Dimensions", "expression": "detail_fields.get('Dimensions')" }, { "line": 266, "receiver": "detail_fields", "key": "Item location", "expression": "detail_fields.get('Item location')" }, { "line": 267, "receiver": "detail_fields", "key": "Collection", "expression": "detail_fields.get('Collection')" }, { "line": 268, "receiver": "detail_fields", "key": "Department", "expression": "detail_fields.get('Department')" }, { "line": 269, "receiver": "detail_fields", "key": "Object number", "expression": "detail_fields.get('Object number')" }, { "line": 273, "receiver": "detail_fields", "key": "Description", "expression": "detail_fields.get('Description')" }, { "line": 274, "receiver": "detail_fields", "key": "Last update", "expression": "detail_fields.get('Last update')" }, { "line": 237, "receiver": "detail_fields", "key": "Record ID", "expression": "detail_fields.get('Record ID')" }, { "line": 241, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 259, "receiver": "detail_fields", "key": "Creator/Producer", "expression": "detail_fields.get('Creator/Producer')" }, { "line": 263, "receiver": "detail_fields", "key": "Technique", "expression": "detail_fields.get('Technique')" }, { "line": 264, "receiver": "detail_fields", "key": "Materials", "expression": "detail_fields.get('Materials')" }, { "line": 270, "receiver": "detail_fields", "key": "Iconclass", "expression": "detail_fields.get('Iconclass')" }, { "line": 271, "receiver": "detail_fields", "key": "Subjects", "expression": "detail_fields.get('Subjects')" }, { "line": 272, "receiver": "detail_fields", "key": "Musical instruments", "expression": "detail_fields.get('Musical instruments')" }, { "line": 237, "receiver": "hidden_id", "key": "value", "expression": "hidden_id.get('value')" }, { "line": 247, "receiver": "img", "key": "title", "expression": "img.get('title')" }, { "line": 247, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 248, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 222, "expression": "soup.find('link', rel='canonical')" }, { "line": 226, "expression": "soup.select('table.table tr')" }, { "line": 236, "expression": "soup.select_one('input.hiddenId')" }, { "line": 240, "expression": "soup.select('img.recordcover, #mediaTypeIcon img, img.icn')" }, { "line": 227, "expression": "row.find('th')" }, { "line": 228, "expression": "row.find('td')" }, { "line": 238, "expression": "soup.select_one('.title')" }, { "line": 238, "expression": "soup.select_one('.title')" } ], "dictionary_entries": [ { "line": 254, "key": "ridim_id", "value_expression": "parsed_id" }, { "line": 255, "key": "record_number", "value_expression": "detail_fields.get('Record ID')" }, { "line": 256, "key": "ridim_siglum", "value_expression": "detail_fields.get('RIdIM siglum')" }, { "line": 257, "key": "title", "value_expression": "title" }, { "line": 258, "key": "creator", "value_expression": "detail_fields.get('Creator/Producer')" }, { "line": 259, "key": "creators", "value_expression": "_split_terms(detail_fields.get('Creator/Producer'))" }, { "line": 260, "key": "production_date", "value_expression": "detail_fields.get('Production date')" }, { "line": 261, "key": "category", "value_expression": "detail_fields.get('Category')" }, { "line": 262, "key": "object_type", "value_expression": "detail_fields.get('Object type')" }, { "line": 263, "key": "technique", "value_expression": "_split_terms(detail_fields.get('Technique'))" }, { "line": 264, "key": "materials", "value_expression": "_split_terms(detail_fields.get('Materials'))" }, { "line": 265, "key": "dimensions", "value_expression": "detail_fields.get('Dimensions')" }, { "line": 266, "key": "item_location", "value_expression": "detail_fields.get('Item location')" }, { "line": 267, "key": "collection", "value_expression": "detail_fields.get('Collection')" }, { "line": 268, "key": "department", "value_expression": "detail_fields.get('Department')" }, { "line": 269, "key": "object_number", "value_expression": "detail_fields.get('Object number')" }, { "line": 270, "key": "iconclass", "value_expression": "_split_terms(detail_fields.get('Iconclass'))" }, { "line": 271, "key": "subjects", "value_expression": "_split_terms(detail_fields.get('Subjects'))" }, { "line": 272, "key": "musical_instruments", "value_expression": "_split_terms(detail_fields.get('Musical instruments'))" }, { "line": 273, "key": "description", "value_expression": "detail_fields.get('Description')" }, { "line": 274, "key": "last_update", "value_expression": "detail_fields.get('Last update')" }, { "line": 275, "key": "detail_fields", "value_expression": "detail_fields" }, { "line": 276, "key": "images", "value_expression": "images" }, { "line": 277, "key": "url", "value_expression": "url" }, { "line": 278, "key": "source_url", "value_expression": "url" }, { "line": 279, "key": "canonical_url", "value_expression": "canonical" }, { "line": 280, "key": "raw_html", "value_expression": "html" }, { "line": 281, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 246, "key": "url", "value_expression": "urljoin(url or self.base_url, str(src))" }, { "line": 247, "key": "title", "value_expression": "img.get('title') or img.get('alt') or 'RIdIM image'" }, { "line": 248, "key": "label", "value_expression": "img.get('title') or 'RIdIM image'" }, { "line": 249, "key": "status", "value_expression": "'discovered'" } ], "subscript_assignments": [] }, { "symbol": "RidimIconographyCrawler.parse_rss", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 165, "line_end": 209, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 167, "expression": "root.find('channel')" } ], "dictionary_entries": [ { "line": 171, "key": "opensearch", "value_expression": "'http://a9.com/-/spec/opensearch/1.1/'" }, { "line": 172, "key": "dc", "value_expression": "'http://purl.org/dc/elements/1.1/'" }, { "line": 187, "key": "ridim_id", "value_expression": "ridim_id" }, { "line": 188, "key": "title", "value_expression": "title" }, { "line": 189, "key": "creator", "value_expression": "creator" }, { "line": 190, "key": "creators", "value_expression": "[creator] if creator else []" }, { "line": 191, "key": "production_date", "value_expression": "production_date" }, { "line": 192, "key": "url", "value_expression": "link" }, { "line": 193, "key": "source_url", "value_expression": "link" }, { "line": 194, "key": "rss_url", "value_expression": "rss_url" }, { "line": 195, "key": "rss_item", "value_expression": "{'title': title, 'link': link, 'guid': _clean_text(item.findtext('guid')), 'author': _clean_text(item.findtext('author')), 'dc_date': production_date, 'dc_creator': creator, 'pub_date': _clean_text(item.findtext('pubDate'))}" }, { "line": 204, "key": "status", "value_expression": "'initiated'" }, { "line": 205, "key": "result_index", "value_expression": "index" }, { "line": 206, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 196, "key": "title", "value_expression": "title" }, { "line": 197, "key": "link", "value_expression": "link" }, { "line": 198, "key": "guid", "value_expression": "_clean_text(item.findtext('guid'))" }, { "line": 199, "key": "author", "value_expression": "_clean_text(item.findtext('author'))" }, { "line": 200, "key": "dc_date", "value_expression": "production_date" }, { "line": 201, "key": "dc_creator", "value_expression": "creator" }, { "line": 202, "key": "pub_date", "value_expression": "_clean_text(item.findtext('pubDate'))" } ], "subscript_assignments": [] }, { "symbol": "RidimIconographyCrawler.preflight_crawl", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 82, "line_end": 92, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 87, "key": "ok", "value_expression": "True" }, { "line": 88, "key": "query", "value_expression": "options['query']" }, { "line": 89, "key": "max_pages", "value_expression": "options['max_pages']" }, { "line": 90, "key": "limit", "value_expression": "options['limit']" }, { "line": 91, "key": "fetch_details", "value_expression": "options['fetch_details']" }, { "line": 85, "key": "ok", "value_expression": "False" }, { "line": 85, "key": "message", "value_expression": "'No RIdIM query was configured.'" } ], "subscript_assignments": [] }, { "symbol": "RidimIconographyCrawler.scrape_row_to_transfer_record", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 365, "line_end": 381, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [ { "line": 374, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 378, "receiver": "raw_payload", "key": "raw_html", "expression": "raw_payload.get('raw_html')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "RidimIconographyCrawler.transform", "path": "aggregator/ridim_iconography_crawler.py", "line_start": 284, "line_end": 300, "inherited": false, "file_sha256": "985fa8604c796f3e47266c7254b20e4f0b2734e86020168448ec13e506ee73c9", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 298, "receiver": "payload", "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 299, "receiver": "payload", "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 297, "receiver": "payload", "key": "raw_html", "expression": "payload.get('raw_html')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "RidimIconographyCrawler.transform", "path": "aggregator/ridim_iconography_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 286, "kind": "input read" }, { "symbol": "RidimIconographyCrawler.transform", "path": "aggregator/ridim_iconography_crawler.py", "key": "crawl_run_id", "value_expression": "payload.get('crawl_run_id')", "line": 298, "kind": "input read" }, { "symbol": "RidimIconographyCrawler.transform", "path": "aggregator/ridim_iconography_crawler.py", "key": "transform_run_id", "value_expression": "payload.get('transform_run_id')", "line": 299, "kind": "input read" }, { "symbol": "RidimIconographyCrawler.transform", "path": "aggregator/ridim_iconography_crawler.py", "key": "raw_html", "value_expression": "payload.get('raw_html')", "line": 297, "kind": "input read" }, { "symbol": "RidimIconographyCrawler.crawl", "path": "aggregator/ridim_iconography_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 96, "kind": "input read" }, { "symbol": "RidimIconographyCrawler.fetch_detail", "path": "aggregator/ridim_iconography_crawler.py", "key": "timeout", "value_expression": "self.config.get('timeout', 30)", "line": 214, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/ridim_iconography_crawler.py", "symbol": "RidimIconographyCrawler.transform", "line_start": 284, "line_end": 291, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n payload = dict(raw_data or {})\n raw_json = payload.get(\"raw_data_json\")\n if isinstance(raw_json, str) and raw_json.strip():\n try:\n decoded = json.loads(raw_json)\n if isinstance(decoded, dict):\n decoded.update({key: value for key, value in payload.items() if value not in (None, \"\")})" }, { "path": "aggregator/ridim_iconography_crawler.py", "symbol": "RidimIconographyCrawler.transform", "line_start": 295, "line_end": 300, "text": " return normalize_ridim_iconography_record(\n payload,\n raw_content=payload.get(\"raw_html\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/ridim_iconography_crawler.py", "symbol": "RidimIconographyCrawler.transform", "line_start": 296, "line_end": 300, "text": " payload,\n raw_content=payload.get(\"raw_html\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/ridim_iconography_crawler.py", "symbol": "RidimIconographyCrawler.transform", "line_start": 294, "line_end": 300, "text": " pass\n return normalize_ridim_iconography_record(\n payload,\n raw_content=payload.get(\"raw_html\") or \"\",\n crawl_run_id=payload.get(\"crawl_run_id\"),\n transform_run_id=payload.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/ridim_iconography_crawler.py", "symbol": "RidimIconographyCrawler.crawl", "line_start": 94, "line_end": 101, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n task_id = kwargs.get(\"task_id\")\n total_steps = max(1, options[\"max_pages\"])\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting RIdIM iconography crawl.\", 0, total_steps)\n records_by_id: Dict[str, Dict[str, Any]] = {}\n declared_total = 0\n exhausted = False" }, { "path": "aggregator/ridim_iconography_crawler.py", "symbol": "RidimIconographyCrawler.fetch_detail", "line_start": 211, "line_end": 217, "text": " def fetch_detail(self, url_or_id: str) -> Dict[str, Any]:\n ridim_id = canonical_ridim_id(url_or_id)\n url = url_or_id if str(url_or_id).startswith(\"http\") else urljoin(self.base_url, f\"Record/{ridim_id}\")\n response = self.session.get(url, timeout=int(self.config.get(\"timeout\", 30)))\n response.raise_for_status()\n html = response.content.decode(getattr(response, \"encoding\", None) or \"utf-8\", errors=\"replace\")\n return self.parse_detail_html(html, url=url, ridim_id=ridim_id)" } ] } }, { "catalogue_id": "S046", "source_key": "sakralorgelforum", "name": "Sakralorgelforum", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Sakralorgelforum", "scope_description": "Sacred music, organs, playing, liturgy and community discussion", "documentary_unit": "Discussion thread and dated post", "source_description": "The public index includes instrument and musical discussions alongside liturgy, community and marketplace sections. It exposes recent activity and topic counts, but these interface statistics do not establish a complete exportable corpus.", "source_description_references": [ "sakralorgelforum" ], "interpretation": "Discussion is useful for discovery and reception research. Assertions about instruments need attribution to the particular post and corroboration appropriate to the question. Participant prose and personal labels are excluded from this package.", "rights_observation": "No general permission to redistribute posts or attachments was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "sakralorgelforum" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The subclass uses the forum-thread family with its source-specific configuration. The effective inherited methods are listed explicitly in this atlas.", "transformation": "The common transform combines acquired thread pages, preserves listing context and creates a thread record with nested posts. Its inheritance should be described directly rather than pretending that a separate class supplies an independent schema. Source-qualified thread/post identities remain distinct from people or instruments mentioned.", "limitations": "The family's pagination and quote-attribution limitations also apply here. A successful common parser does not prove that every source-specific theme or historical layout is supported. Author profile and reaction metadata are documentary context, not an organological reliability score." }, "implementation": { "class": "SakralorgelForumCrawler", "class_line": 924, "path": "aggregator/kirchenorgelforum_crawler.py", "inheritance": [ "SakralorgelForumCrawler", "KirchenorgelForumCrawler", "BaseCrawler" ], "configuration": { "record_label": { "value": "threads", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 44, "class": "KirchenorgelForumCrawler" }, "transformed_json_id_column": { "value": "thread_id", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 46, "class": "KirchenorgelForumCrawler" }, "source_key": { "value": "sakralorgelforum", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 927, "class": "SakralorgelForumCrawler" }, "base_url": { "value": "https://www.sakralorgelforum.net", "path": "aggregator/kirchenorgelforum_crawler.py", "line": 928, "class": "SakralorgelForumCrawler" } }, "methods": [ { "symbol": "KirchenorgelForumCrawler._extract_images", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 801, "line_end": 836, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 806, "receiver": "image", "key": "src", "expression": "image.get('src')" }, { "line": 820, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 814, "receiver": "image", "key": "alt", "expression": "image.get('alt')" }, { "line": 814, "receiver": "image", "key": "title", "expression": "image.get('title')" } ], "html_queries": [ { "line": 805, "expression": "node.select('img[src]')" }, { "line": 819, "expression": "node.select('a[href]')" } ], "dictionary_entries": [ { "line": 809, "key": "url", "value_expression": "url" }, { "line": 810, "key": "link", "value_expression": "url" }, { "line": 811, "key": "file_name", "value_expression": "os.path.basename(urlparse(url).path) or f'post_{post_id}_{index}.jpg'" }, { "line": 812, "key": "post_id", "value_expression": "post_id" }, { "line": 813, "key": "credit", "value_expression": "author" }, { "line": 814, "key": "caption", "value_expression": "image.get('alt') or image.get('title') or ''" }, { "line": 815, "key": "source_role", "value_expression": "'post_embedded_image'" }, { "line": 816, "key": "status", "value_expression": "'discovered'" }, { "line": 826, "key": "url", "value_expression": "url" }, { "line": 827, "key": "link", "value_expression": "url" }, { "line": 828, "key": "file_name", "value_expression": "os.path.basename(urlparse(url).path) or f'post_{post_id}_{index}.jpg'" }, { "line": 829, "key": "post_id", "value_expression": "post_id" }, { "line": 830, "key": "credit", "value_expression": "author" }, { "line": 831, "key": "caption", "value_expression": "link.get_text(' ', strip=True)" }, { "line": 832, "key": "source_role", "value_expression": "'post_linked_image'" }, { "line": 833, "key": "status", "value_expression": "'discovered'" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._extract_links", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 783, "line_end": 799, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 788, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 787, "expression": "node.select('a[href]')" } ], "dictionary_entries": [ { "line": 793, "key": "url", "value_expression": "urljoin(source_url or self.base_url, href)" }, { "line": 794, "key": "label", "value_expression": "link.get_text(' ', strip=True)" }, { "line": 795, "key": "raw_href", "value_expression": "href" }, { "line": 796, "key": "kind", "value_expression": "'image' if _looks_like_image(href) else 'document' if _looks_like_document(href) else 'internal' if href.startswith('t') or href.startswith('#') else 'external'" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._extract_quotes", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 838, "line_end": 858, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 844, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 842, "expression": "node.select('blockquote')" }, { "line": 843, "expression": "quote.select_one('a[href*=\"#msg\"]')" } ], "dictionary_entries": [ { "line": 851, "key": "type", "value_expression": "'quote'" }, { "line": 852, "key": "post_id", "value_expression": "post_id" }, { "line": 853, "key": "quoted_post_id", "value_expression": "quoted_post_id" }, { "line": 854, "key": "quoted_url", "value_expression": "urljoin(source_url or self.base_url, href) if href else ''" }, { "line": 855, "key": "text", "value_expression": "quote.get_text('\\n', strip=True)" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._extract_reactions", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 861, "line_end": 878, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "startswith" ], "literal_get_calls": [ { "line": 865, "receiver": "user", "key": "class", "expression": "user.get('class')" } ], "html_queries": [ { "line": 863, "expression": "box.select('.postratings .ratinguser')" } ], "dictionary_entries": [ { "line": 872, "key": "type", "value_expression": "'thanks'" }, { "line": 873, "key": "post_id", "value_expression": "post_id" }, { "line": 874, "key": "user_display_name", "value_expression": "user_name" }, { "line": 875, "key": "user_source_id", "value_expression": "user_id" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._media_images_from_posts", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 907, "line_end": 911, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 910, "receiver": "post", "key": "images", "expression": "post.get('images')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._parse_forum_thread_rows", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 571, "line_end": 618, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_forum_id", "_thread_parts" ], "literal_get_calls": [ { "line": 593, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 586, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 582, "expression": "soup.select('a.topiclink, a.threadlink')" }, { "line": 579, "expression": "soup.select_one('h1, .forum-title, .page-title')" }, { "line": 585, "expression": "soup.select('tr.threadrow a[href]')" }, { "line": 575, "expression": "soup.select('#breadcrumbs a, #breadcrumbs strong')" } ], "dictionary_entries": [ { "line": 605, "key": "thread_id", "value_expression": "thread_id" }, { "line": 606, "key": "forum_id", "value_expression": "parsed_forum_id or forum_id" }, { "line": 607, "key": "forum_title", "value_expression": "forum_title" }, { "line": 608, "key": "title", "value_expression": "_dedupe_repeated_label(link.get_text(' ', strip=True))" }, { "line": 609, "key": "author", "value_expression": "author" }, { "line": 610, "key": "reply_count", "value_expression": "_first_int(_select_text(container, '.fposts'))" }, { "line": 611, "key": "view_count", "value_expression": "_views_from_text(text)" }, { "line": 612, "key": "last_post_author", "value_expression": "_last_author_from_container(container) or _last_author_from_text(text)" }, { "line": 613, "key": "last_post_at_raw", "value_expression": "_last_date_from_container(container) or _last_date_from_text(text)" }, { "line": 614, "key": "url", "value_expression": "url" }, { "line": 615, "key": "raw_index_html", "value_expression": "str(container) if container else str(link)" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._parse_post", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 522, "line_end": 569, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_extract_images", "_extract_links", "_extract_quotes", "_extract_reactions", "_parse_user_meta" ], "literal_get_calls": [ { "line": 554, "receiver": "profile_node", "key": "href", "expression": "profile_node.get('href')" }, { "line": 554, "receiver": "profile_node", "key": "href", "expression": "profile_node.get('href')" }, { "line": 555, "receiver": "avatar_node", "key": "src", "expression": "avatar_node.get('src')" }, { "line": 555, "receiver": "avatar_node", "key": "src", "expression": "avatar_node.get('src')" }, { "line": 567, "receiver": "quote", "key": "quoted_post_id", "expression": "quote.get('quoted_post_id')" }, { "line": 529, "receiver": "box", "key": "id", "expression": "box.get('id')" }, { "line": 527, "receiver": "anchor", "key": "name", "expression": "anchor.get('name')" }, { "line": 527, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 554, "receiver": "profile_node", "key": "href", "expression": "profile_node.get('href', '')" } ], "html_queries": [ { "line": 541, "expression": "box.select_one('.xquoteable')" }, { "line": 542, "expression": "box.select_one('.msignature')" }, { "line": 523, "expression": "box.select_one('.boxheader')" }, { "line": 523, "expression": "box.select_one('.card-header')" }, { "line": 524, "expression": "header.select_one('a[name^=\"msg\"]')" }, { "line": 526, "expression": "box.select_one('a[name^=\"msg\"], a[href^=\"#msg\"]')" }, { "line": 531, "expression": "header.select('a strong')" }, { "line": 533, "expression": "header.select_one('.mposttime')" }, { "line": 535, "expression": "header.select_one('.float-right')" }, { "line": 537, "expression": "box.select_one('.messageUser strong')" }, { "line": 537, "expression": "box.select_one('.muser .userlink')" }, { "line": 537, "expression": "box.select_one('.xsuser .userlink')" }, { "line": 539, "expression": "box.select_one('.messageUser a[href]')" }, { "line": 539, "expression": "box.select_one('.muser a[href], .xsuser a[href]')" }, { "line": 540, "expression": "box.select_one('.messageAvatar img')" }, { "line": 540, "expression": "box.select_one('.image_container img, .xsuser img')" } ], "dictionary_entries": [ { "line": 549, "key": "post_id", "value_expression": "post_id" }, { "line": 550, "key": "post_number", "value_expression": "post_number" }, { "line": 551, "key": "url", "value_expression": "f'{source_url}#msg{post_id}' if post_id else source_url" }, { "line": 552, "key": "title", "value_expression": "title" }, { "line": 553, "key": "author_display_name", "value_expression": "author" }, { "line": 554, "key": "author_profile_url", "value_expression": "urljoin(self.base_url, profile_node.get('href')) if profile_node and profile_node.get('href') and (not profile_node.get('href', '').startswith('#')) else ''" }, { "line": 555, "key": "author_avatar_url", "value_expression": "_abs_url(avatar_node.get('src'), self.base_url) if avatar_node and avatar_node.get('src') else ''" }, { "line": 556, "key": "created_at", "value_expression": "_parse_forum_datetime(raw_time)" }, { "line": 557, "key": "created_at_raw", "value_expression": "raw_time" }, { "line": 558, "key": "edited_at_raw", "value_expression": "_edited_text(raw_time)" }, { "line": 559, "key": "html", "value_expression": "str(body_node) if body_node else ''" }, { "line": 560, "key": "text", "value_expression": "body_node.get_text('\\n', strip=True) if body_node else ''" }, { "line": 561, "key": "signature_html", "value_expression": "str(signature_node) if signature_node else ''" }, { "line": 562, "key": "signature_text", "value_expression": "signature_node.get_text('\\n', strip=True) if signature_node else ''" }, { "line": 563, "key": "author_metadata", "value_expression": "user_meta" }, { "line": 564, "key": "links", "value_expression": "links" }, { "line": 565, "key": "images", "value_expression": "images" }, { "line": 566, "key": "quotes", "value_expression": "quotes" }, { "line": 567, "key": "responds_to", "value_expression": "[quote['quoted_post_id'] for quote in quotes if quote.get('quoted_post_id')]" }, { "line": 568, "key": "reactions", "value_expression": "reactions" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler._parse_user_meta", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 768, "line_end": 781, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [], "html_queries": [ { "line": 770, "expression": "box.select('.messageinformation tr')" }, { "line": 775, "expression": "box.select('.userdetails .row')" }, { "line": 771, "expression": "row.find_all('td')" }, { "line": 776, "expression": "row.select_one('.minfo_head')" }, { "line": 777, "expression": "row.select_one('.minfo_cont')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.crawl", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 156, "line_end": 265, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_discover_forums", "_forum_page_url", "_parse_forum_thread_rows", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 161, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 158, "receiver": "kwargs", "key": "max_forum_pages", "expression": "kwargs.get('max_forum_pages')" }, { "line": 158, "receiver": "self.config", "key": "max_forum_pages", "expression": "self.config.get('max_forum_pages', 1)" }, { "line": 159, "receiver": "kwargs", "key": "max_threads", "expression": "kwargs.get('max_threads')" }, { "line": 159, "receiver": "self.config", "key": "max_threads", "expression": "self.config.get('max_threads', 100)" }, { "line": 160, "receiver": "kwargs", "key": "forum_urls", "expression": "kwargs.get('forum_urls')" }, { "line": 160, "receiver": "self.config", "key": "forum_urls", "expression": "self.config.get('forum_urls')" }, { "line": 231, "receiver": "row", "key": "thread_id", "expression": "row.get('thread_id')" }, { "line": 232, "receiver": "row", "key": "forum_id", "expression": "row.get('forum_id')" }, { "line": 233, "receiver": "row", "key": "forum_title", "expression": "row.get('forum_title')" }, { "line": 234, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 235, "receiver": "row", "key": "author", "expression": "row.get('author')" }, { "line": 236, "receiver": "row", "key": "reply_count", "expression": "row.get('reply_count')" }, { "line": 237, "receiver": "row", "key": "view_count", "expression": "row.get('view_count')" }, { "line": 238, "receiver": "row", "key": "last_post_author", "expression": "row.get('last_post_author')" }, { "line": 239, "receiver": "row", "key": "last_post_at_raw", "expression": "row.get('last_post_at_raw')" }, { "line": 240, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 242, "receiver": "row", "key": "raw_index_html", "expression": "row.get('raw_index_html')" }, { "line": 250, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.5)" }, { "line": 190, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.normalized_batch_record", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 432, "line_end": 453, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 450, "receiver": "transformed.get('provenance', {}).get('identity', {})", "key": "crawl_run_id", "expression": "transformed.get('provenance', {}).get('identity', {}).get('crawl_run_id', '')" }, { "line": 450, "receiver": "transformed.get('provenance', {})", "key": "identity", "expression": "transformed.get('provenance', {}).get('identity', {})" }, { "line": 450, "receiver": "transformed", "key": "provenance", "expression": "transformed.get('provenance', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.parse_thread_html", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 455, "line_end": 459, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "parse_thread_pages" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 459, "key": "url", "value_expression": "source_url" }, { "line": 459, "key": "html", "value_expression": "html" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.parse_thread_pages", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 461, "line_end": 520, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "_interactions_from_posts", "_links_from_posts", "_media_images_from_posts", "_parse_post", "_participants_from_posts", "_thread_parts" ], "literal_get_calls": [ { "line": 465, "receiver": "pages[0]", "key": "url", "expression": "pages[0].get('url')" }, { "line": 466, "receiver": "pages[0]", "key": "html", "expression": "pages[0].get('html')" }, { "line": 483, "receiver": "page", "key": "url", "expression": "page.get('url')" }, { "line": 488, "receiver": "post", "key": "post_id", "expression": "post.get('post_id')" }, { "line": 484, "receiver": "page", "key": "html", "expression": "page.get('html')" }, { "line": 509, "receiver": "posts[0]", "key": "created_at", "expression": "posts[0].get('created_at', '')" }, { "line": 510, "receiver": "posts[-1]", "key": "created_at", "expression": "posts[-1].get('created_at', '')" }, { "line": 511, "receiver": "posts[0]", "key": "author_display_name", "expression": "posts[0].get('author_display_name', '')" }, { "line": 496, "receiver": "link", "key": "url", "expression": "link.get('url', '')" }, { "line": 518, "receiver": "page", "key": "url", "expression": "page.get('url')" } ], "html_queries": [ { "line": 469, "expression": "first_soup.select('#breadcrumbs a, #breadcrumbs strong')" }, { "line": 485, "expression": "soup.select('#messagewrap > div.box')" }, { "line": 485, "expression": "soup.select('div.message[id^=\"post_\"]')" } ], "dictionary_entries": [ { "line": 498, "key": "forum", "value_expression": "{'forum_id': forum_id, 'category_title': category_title, 'forum_title': forum_title, 'url': urljoin(self.base_url, f'f{forum_id}.html') if forum_id else ''}" }, { "line": 504, "key": "thread", "value_expression": "{'thread_id': thread_id, 'title': thread_title, 'url': source_url or first_url, 'post_count': len(posts), 'created_at': posts[0].get('created_at', '') if posts else '', 'updated_at': posts[-1].get('created_at', '') if posts else '', 'starter': posts[0].get('author_display_name', '') if posts else ''}" }, { "line": 513, "key": "posts", "value_expression": "posts" }, { "line": 514, "key": "participants", "value_expression": "participants" }, { "line": 515, "key": "interactions", "value_expression": "interactions" }, { "line": 516, "key": "links", "value_expression": "links" }, { "line": 517, "key": "media", "value_expression": "{'images': media_images, 'documents': documents}" }, { "line": 518, "key": "source_snapshot", "value_expression": "{'page_urls': [page['url'] for page in pages if page.get('url')]}" }, { "line": 519, "key": "quality", "value_expression": "{'completeness': {'thread_pages_crawled': len(pages)}}" }, { "line": 499, "key": "forum_id", "value_expression": "forum_id" }, { "line": 500, "key": "category_title", "value_expression": "category_title" }, { "line": 501, "key": "forum_title", "value_expression": "forum_title" }, { "line": 502, "key": "url", "value_expression": "urljoin(self.base_url, f'f{forum_id}.html') if forum_id else ''" }, { "line": 505, "key": "thread_id", "value_expression": "thread_id" }, { "line": 506, "key": "title", "value_expression": "thread_title" }, { "line": 507, "key": "url", "value_expression": "source_url or first_url" }, { "line": 508, "key": "post_count", "value_expression": "len(posts)" }, { "line": 509, "key": "created_at", "value_expression": "posts[0].get('created_at', '') if posts else ''" }, { "line": 510, "key": "updated_at", "value_expression": "posts[-1].get('created_at', '') if posts else ''" }, { "line": 511, "key": "starter", "value_expression": "posts[0].get('author_display_name', '') if posts else ''" }, { "line": 517, "key": "images", "value_expression": "media_images" }, { "line": 517, "key": "documents", "value_expression": "documents" }, { "line": 518, "key": "page_urls", "value_expression": "[page['url'] for page in pages if page.get('url')]" }, { "line": 519, "key": "completeness", "value_expression": "{'thread_pages_crawled': len(pages)}" }, { "line": 464, "key": "url", "value_expression": "source_url" }, { "line": 464, "key": "html", "value_expression": "''" }, { "line": 519, "key": "thread_pages_crawled", "value_expression": "len(pages)" } ], "subscript_assignments": [ { "line": 489, "value_line": 489, "key": "page_index", "target_expression": "post['page_index']", "value_expression": "page_index" } ] }, { "symbol": "KirchenorgelForumCrawler.scrape_row_to_transfer_record", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 396, "line_end": 430, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [], "literal_get_calls": [ { "line": 410, "receiver": "row", "key": "view_count", "expression": "row.get('view_count')" }, { "line": 407, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 408, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 411, "receiver": "row", "key": "last_post_at_raw", "expression": "row.get('last_post_at_raw')" }, { "line": 412, "receiver": "row", "key": "author", "expression": "row.get('author')" }, { "line": 415, "receiver": "row", "key": "forum_id", "expression": "row.get('forum_id')" }, { "line": 416, "receiver": "row", "key": "forum_title", "expression": "row.get('forum_title')" }, { "line": 417, "receiver": "row", "key": "forum_url", "expression": "row.get('forum_url')" }, { "line": 419, "receiver": "row", "key": "scraped_at", "expression": "row.get('scraped_at')" }, { "line": 419, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 425, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 426, "receiver": "row", "key": "raw_html", "expression": "row.get('raw_html')" }, { "line": 426, "receiver": "row", "key": "raw_index_html", "expression": "row.get('raw_index_html')" }, { "line": 409, "receiver": "row", "key": "post_count", "expression": "row.get('post_count')" } ], "html_queries": [], "dictionary_entries": [ { "line": 405, "key": "thread", "value_expression": "{'thread_id': str(row.get(id_column) or ''), 'title': row.get('title') or '', 'url': row.get('url') or '', 'post_count': int(row.get('post_count') or 0), 'view_count': row.get('view_count'), 'updated_at': row.get('last_post_at_raw') or '', 'starter': row.get('author') or ''}" }, { "line": 414, "key": "forum", "value_expression": "{'forum_id': row.get('forum_id') or '', 'forum_title': row.get('forum_title') or '', 'url': row.get('forum_url') or ''}" }, { "line": 419, "key": "source_info", "value_expression": "{'retrieved_at': row.get('scraped_at') or '', 'url': row.get('url') or ''}" }, { "line": 406, "key": "thread_id", "value_expression": "str(row.get(id_column) or '')" }, { "line": 407, "key": "title", "value_expression": "row.get('title') or ''" }, { "line": 408, "key": "url", "value_expression": "row.get('url') or ''" }, { "line": 409, "key": "post_count", "value_expression": "int(row.get('post_count') or 0)" }, { "line": 410, "key": "view_count", "value_expression": "row.get('view_count')" }, { "line": 411, "key": "updated_at", "value_expression": "row.get('last_post_at_raw') or ''" }, { "line": 412, "key": "starter", "value_expression": "row.get('author') or ''" }, { "line": 415, "key": "forum_id", "value_expression": "row.get('forum_id') or ''" }, { "line": 416, "key": "forum_title", "value_expression": "row.get('forum_title') or ''" }, { "line": 417, "key": "url", "value_expression": "row.get('forum_url') or ''" }, { "line": 419, "key": "retrieved_at", "value_expression": "row.get('scraped_at') or ''" }, { "line": 419, "key": "url", "value_expression": "row.get('url') or ''" } ], "subscript_assignments": [] }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "line_start": 340, "line_end": 394, "inherited": true, "file_sha256": "d008e9c0d121af88c1dbc5a710a013e2e2125f08f08e1670b4eb31dfa49b5469", "self_calls": [ "parse_thread_html", "parse_thread_pages" ], "literal_get_calls": [ { "line": 341, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 341, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 345, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 345, "receiver": "parsed.get('thread', {})", "key": "url", "expression": "parsed.get('thread', {}).get('url')" }, { "line": 358, "receiver": "forum", "key": "forum_id", "expression": "forum.get('forum_id')" }, { "line": 359, "receiver": "forum", "key": "forum_title", "expression": "forum.get('forum_title')" }, { "line": 359, "receiver": "raw_data", "key": "forum_title", "expression": "raw_data.get('forum_title')" }, { "line": 361, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 361, "receiver": "parsed['thread']", "key": "url", "expression": "parsed['thread'].get('url')" }, { "line": 365, "receiver": "raw_data", "key": "raw_index_html", "expression": "raw_data.get('raw_index_html')" }, { "line": 342, "receiver": "raw_data", "key": "pages", "expression": "raw_data.get('pages')" }, { "line": 344, "receiver": "raw_data", "key": "thread_id", "expression": "raw_data.get('thread_id')" }, { "line": 344, "receiver": "parsed.get('thread', {})", "key": "thread_id", "expression": "parsed.get('thread', {}).get('thread_id')" }, { "line": 388, "receiver": "parsed['thread']", "key": "thread_id", "expression": "parsed['thread'].get('thread_id')" }, { "line": 389, "receiver": "parsed['thread']", "key": "url", "expression": "parsed['thread'].get('url')" }, { "line": 392, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 393, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" }, { "line": 343, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 343, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 345, "receiver": "parsed", "key": "thread", "expression": "parsed.get('thread', {})" }, { "line": 358, "receiver": "raw_data", "key": "forum_id", "expression": "raw_data.get('forum_id')" }, { "line": 363, "receiver": "page", "key": "url", "expression": "page.get('url')" }, { "line": 344, "receiver": "parsed", "key": "thread", "expression": "parsed.get('thread', {})" }, { "line": 380, "receiver": "raw_data", "key": "forum_url", "expression": "raw_data.get('forum_url')" }, { "line": 381, "receiver": "raw_data", "key": "scraped_at", "expression": "raw_data.get('scraped_at')" }, { "line": 382, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 374, "receiver": "parsed['thread']", "key": "thread_id", "expression": "parsed['thread'].get('thread_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 347, "key": "title", "value_expression": "('title',)" }, { "line": 348, "key": "view_count", "value_expression": "('view_count',)" }, { "line": 349, "key": "starter", "value_expression": "('author',)" }, { "line": 350, "key": "updated_at", "value_expression": "('last_post_at_raw',)" }, { "line": 372, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, str(parsed['thread'].get('thread_id') or 'unknown'), 'forum_index_html')" }, { "line": 377, "key": "document_kind", "value_expression": "'forum_index_html'" }, { "line": 378, "key": "content_sha256", "value_expression": "index_hash" }, { "line": 379, "key": "content_type", "value_expression": "'text/html'" }, { "line": 380, "key": "url", "value_expression": "raw_data.get('forum_url') or ''" }, { "line": 381, "key": "retrieved_at", "value_expression": "raw_data.get('scraped_at') or ''" }, { "line": 382, "key": "crawl_run_id", "value_expression": "raw_data.get('crawl_run_id') or ''" } ], "subscript_assignments": [ { "line": 344, "value_line": 344, "key": "thread_id", "target_expression": "parsed.setdefault('thread', {})['thread_id']", "value_expression": "str(raw_data.get('thread_id') or parsed.get('thread', {}).get('thread_id') or '')" }, { "line": 345, "value_line": 345, "key": "url", "target_expression": "parsed.setdefault('thread', {})['url']", "value_expression": "raw_data.get('url') or parsed.get('thread', {}).get('url') or ''" }, { "line": 358, "value_line": 358, "key": "forum_id", "target_expression": "forum['forum_id']", "value_expression": "forum.get('forum_id') or str(raw_data.get('forum_id') or '')" }, { "line": 359, "value_line": 359, "key": "forum_title", "target_expression": "forum['forum_title']", "value_expression": "forum.get('forum_title') or raw_data.get('forum_title') or ''" }, { "line": 360, "value_line": 360, "key": "retrieved_at", "target_expression": "parsed.setdefault('source_info', {})['retrieved_at']", "value_expression": "datetime.now().astimezone().isoformat()" }, { "line": 361, "value_line": 361, "key": "url", "target_expression": "parsed['source_info']['url']", "value_expression": "raw_data.get('url') or parsed['thread'].get('url') or ''" }, { "line": 363, "value_line": 363, "key": "page_urls", "target_expression": "parsed.setdefault('source_snapshot', {})['page_urls']", "value_expression": "[page['url'] for page in pages if page.get('url')]" }, { "line": 364, "value_line": 364, "key": "thread_pages_crawled", "target_expression": "parsed.setdefault('quality', {}).setdefault('completeness', {})['thread_pages_crawled']", "value_expression": "len(pages)" }, { "line": 369, "value_line": 369, "key": "raw_index_html", "target_expression": "snapshot['raw_index_html']", "value_expression": "raw_index_html" } ] } ], "selected_rule_rows": [ { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "html", "value_expression": "raw_data.get('html')", "line": 341, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "raw_html", "value_expression": "raw_data.get('raw_html')", "line": 341, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "url", "value_expression": "raw_data.get('url')", "line": 345, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "forum_id", "value_expression": "forum.get('forum_id')", "line": 358, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "forum_title", "value_expression": "forum.get('forum_title')", "line": 359, "kind": "input read" }, { "symbol": "KirchenorgelForumCrawler.transform", "path": "aggregator/kirchenorgelforum_crawler.py", "key": "raw_index_html", "value_expression": "raw_data.get('raw_index_html')", "line": 365, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 340, "line_end": 346, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n html = raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\"\n pages = _normalize_page_list(raw_data.get(\"pages\")) or _deserialize_thread_pages(html)\n parsed = self.parse_thread_pages(pages, raw_data.get(\"url\") or \"\") if pages else self.parse_thread_html(html, raw_data.get(\"url\") or \"\")\n parsed.setdefault(\"thread\", {})[\"thread_id\"] = str(raw_data.get(\"thread_id\") or parsed.get(\"thread\", {}).get(\"thread_id\") or \"\")\n parsed.setdefault(\"thread\", {})[\"url\"] = raw_data.get(\"url\") or parsed.get(\"thread\", {}).get(\"url\") or \"\"\n for target_key, source_keys in {" }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 342, "line_end": 350, "text": " pages = _normalize_page_list(raw_data.get(\"pages\")) or _deserialize_thread_pages(html)\n parsed = self.parse_thread_pages(pages, raw_data.get(\"url\") or \"\") if pages else self.parse_thread_html(html, raw_data.get(\"url\") or \"\")\n parsed.setdefault(\"thread\", {})[\"thread_id\"] = str(raw_data.get(\"thread_id\") or parsed.get(\"thread\", {}).get(\"thread_id\") or \"\")\n parsed.setdefault(\"thread\", {})[\"url\"] = raw_data.get(\"url\") or parsed.get(\"thread\", {}).get(\"url\") or \"\"\n for target_key, source_keys in {\n \"title\": (\"title\",),\n \"view_count\": (\"view_count\",),\n \"starter\": (\"author\",),\n \"updated_at\": (\"last_post_at_raw\",)," }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 355, "line_end": 363, "text": " parsed[\"thread\"][target_key] = raw_data[source_key]\n break\n forum = parsed.setdefault(\"forum\", {})\n forum[\"forum_id\"] = forum.get(\"forum_id\") or str(raw_data.get(\"forum_id\") or \"\")\n forum[\"forum_title\"] = forum.get(\"forum_title\") or raw_data.get(\"forum_title\") or \"\"\n parsed.setdefault(\"source_info\", {})[\"retrieved_at\"] = datetime.now().astimezone().isoformat()\n parsed[\"source_info\"][\"url\"] = raw_data.get(\"url\") or parsed[\"thread\"].get(\"url\") or \"\"\n if pages:\n parsed.setdefault(\"source_snapshot\", {})[\"page_urls\"] = [page[\"url\"] for page in pages if page.get(\"url\")]" }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 356, "line_end": 364, "text": " break\n forum = parsed.setdefault(\"forum\", {})\n forum[\"forum_id\"] = forum.get(\"forum_id\") or str(raw_data.get(\"forum_id\") or \"\")\n forum[\"forum_title\"] = forum.get(\"forum_title\") or raw_data.get(\"forum_title\") or \"\"\n parsed.setdefault(\"source_info\", {})[\"retrieved_at\"] = datetime.now().astimezone().isoformat()\n parsed[\"source_info\"][\"url\"] = raw_data.get(\"url\") or parsed[\"thread\"].get(\"url\") or \"\"\n if pages:\n parsed.setdefault(\"source_snapshot\", {})[\"page_urls\"] = [page[\"url\"] for page in pages if page.get(\"url\")]\n parsed.setdefault(\"quality\", {}).setdefault(\"completeness\", {})[\"thread_pages_crawled\"] = len(pages)" }, { "path": "aggregator/kirchenorgelforum_crawler.py", "symbol": "KirchenorgelForumCrawler.transform", "line_start": 362, "line_end": 370, "text": " if pages:\n parsed.setdefault(\"source_snapshot\", {})[\"page_urls\"] = [page[\"url\"] for page in pages if page.get(\"url\")]\n parsed.setdefault(\"quality\", {}).setdefault(\"completeness\", {})[\"thread_pages_crawled\"] = len(pages)\n raw_index_html = raw_data.get(\"raw_index_html\") or \"\"\n if raw_index_html:\n index_hash = provenance.content_sha256(raw_index_html)\n snapshot = parsed.setdefault(\"source_snapshot\", {})\n snapshot[\"raw_index_html\"] = raw_index_html\n snapshot.setdefault(\"documents\", []).append(" } ] } }, { "catalogue_id": "S047", "source_key": "sonora", "name": "SONORA", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Göteborg International Organ Academy Association; University of Gothenburg / GRIDH", "scope_description": "Swedish historical organs, primarily preserved instruments built before the 1860s", "documentary_unit": "Archival item linked to organ, building, place or builder", "source_description": "SONORA makes material from six Swedish archives accessible through map, gallery and archive views. Its stated selection follows Einar Erici's inventory and requires substantial original construction to survive; organs retaining only a facade are excluded.", "source_description_references": [ "browser_sonora", "sonora_project", "record_sonora" ], "interpretation": "This survival criterion matters in comparison with inventories that include lost instruments. Archive, series and item references should accompany organ links. The portal's research-project link returned 404 even though the database interface remained accessible.", "rights_observation": "A CC BY 4.0 notice accompanies a specific linked text; no single licence for all six archives was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "browser_sonora", "sonora_project", "record_sonora" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_sonora", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "JSON response linking an organ/work record to place, historical activities and archival documents", "repeated_information": "Numbered activity objects and document objects accompany technical fields; specification is an embedded HTML fragment.", "date_and_state_distinctions": "Historical activities use separate organ/work identifiers, dates and agents for one place.", "interpretive_limit": "The JSON mixes record types and embeds HTML. A document count does not prove that document images were inspected.", "observations": [ "Place name and a distinct place identifier are returned.", "Historical maker and later intervention agents are represented.", "Construction, redisposition, restoration and inventory have separate activity records.", "Embedded HTML contains division stop names and pitches.", "Manual and pedal compasses are included in the specification fragment.", "Separate key and stop action fields are populated.", "Bellows type, quantity and descriptive text are populated.", "Console position and surviving facade-pipe information are present.", "Six archival document references include archive names and item identifiers.", "Historical activity records link to separate organ/work identifiers." ], "uninspected_or_unfilled_material": [], "narrative": "The inherited GOArt endpoint for work 14303 returns numbered historical activities and archival references alongside technical fields. Different work identifiers relate to one place, and the specification is embedded as HTML in the JSON response. The returned archival item references were inspected as metadata; document images were not examined. Place, work and document identifiers must not be collapsed into one record unit." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "Place discovery and organ-detail retrieval are separate routes. Although housed in the static-crawler module, this subclass interprets JSON organ data together with cached place context and coordinates.", "transformation": "The transform reads source organ fields, translates place labels into building/town context, parses builder/history and disposition material, and retains the organ response, place response and reference context together as raw evidence. Structured responses can still contain HTML specifications and documentary references.", "limitations": "The class name or module location does not determine the source format. Place IDs, organ/work IDs and documentary references must be distinguished. A joint raw-evidence object preserves the acquisition context but is an assembled representation, not necessarily a single source response." }, "implementation": { "class": "SonoraCrawler", "class_line": 2083, "path": "aggregator/static_html_crawler.py", "inheritance": [ "SonoraCrawler", "StaticHtmlOrganCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "sonora", "path": "aggregator/static_html_crawler.py", "line": 2084, "class": "SonoraCrawler" }, "id_column": { "value": "sonora_id", "path": "aggregator/static_html_crawler.py", "line": 2085, "class": "SonoraCrawler" }, "default_country": { "value": "Sweden", "path": "aggregator/static_html_crawler.py", "line": 2089, "class": "SonoraCrawler" }, "default_region": { "value": "", "path": "aggregator/static_html_crawler.py", "line": 29, "class": "StaticHtmlOrganCrawler" }, "start_urls": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 30, "class": "StaticHtmlOrganCrawler" }, "detail_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 31, "class": "StaticHtmlOrganCrawler" }, "exclude_url_patterns": { "value": [], "path": "aggregator/static_html_crawler.py", "line": 32, "class": "StaticHtmlOrganCrawler" }, "BUILDER_LABELS": { "value": [ "builder", "organ builder", "organbuilder", "orgelbouwer", "orgelbauer", "facteur", "facteur d'orgue", "maker" ], "path": "aggregator/static_html_crawler.py", "line": 34, "class": "StaticHtmlOrganCrawler" }, "YEAR_LABELS": { "value": [ "year", "date", "built", "installed", "baujahr", "bouwjaar", "construction" ], "path": "aggregator/static_html_crawler.py", "line": 44, "class": "StaticHtmlOrganCrawler" }, "base_url": { "value": "https://sonora.dh.gu.se", "path": "aggregator/static_html_crawler.py", "line": 2087, "class": "SonoraCrawler" }, "api_base_url": { "value": "https://orgeldatabas.gu.se/webgoart/goart", "path": "aggregator/static_html_crawler.py", "line": 2088, "class": "SonoraCrawler" } }, "methods": [ { "symbol": "StaticHtmlOrganCrawler._build_involved", "path": "aggregator/static_html_crawler.py", "line_start": 771, "line_end": 781, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 776, "key": "name", "value_expression": "str(builder)" }, { "line": 777, "key": "activities", "value_expression": "['builder']" }, { "line": 778, "key": "dates", "value_expression": "[str(year)] if year else []" }, { "line": 779, "key": "raw_str", "value_expression": "str(builder)" } ], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._builder_name", "path": "aggregator/static_html_crawler.py", "line_start": 2299, "line_end": 2301, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._clean_label", "path": "aggregator/static_html_crawler.py", "line_start": 850, "line_end": 851, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._division_starts_line", "path": "aggregator/static_html_crawler.py", "line_start": 697, "line_end": 702, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._event_for_organ", "path": "aggregator/static_html_crawler.py", "line_start": 2345, "line_end": 2362, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_builder_name", "_clean_text" ], "literal_get_calls": [ { "line": 2346, "receiver": "organ_data", "key": "org_id", "expression": "organ_data.get('org_id')" }, { "line": 2356, "receiver": "event", "key": "involved", "expression": "event.get('involved')" }, { "line": 2358, "receiver": "event", "key": "type", "expression": "event.get('type', '')" }, { "line": 2360, "receiver": "involved[0]", "key": "name", "expression": "involved[0].get('name', '')" }, { "line": 2349, "receiver": "item", "key": "org_nr", "expression": "item.get('org_nr')" }, { "line": 2351, "receiver": "item", "key": "work", "expression": "item.get('work', '')" }, { "line": 2352, "receiver": "item", "key": "date", "expression": "item.get('date', '')" }, { "line": 2353, "receiver": "item", "key": "builder", "expression": "item.get('builder', '')" }, { "line": 2359, "receiver": "event", "key": "dates", "expression": "event.get('dates')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2358, "key": "work", "value_expression": "event.get('type', '')" }, { "line": 2359, "key": "date", "value_expression": "(event.get('dates') or [''])[0]" }, { "line": 2360, "key": "builder_name", "value_expression": "involved[0].get('name', '') if involved else ''" }, { "line": 2351, "key": "work", "value_expression": "self._clean_text(item.get('work', ''))" }, { "line": 2352, "key": "date", "value_expression": "self._clean_text(item.get('date', ''))" }, { "line": 2353, "key": "builder_name", "value_expression": "self._builder_name(item.get('builder', ''))" } ], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._events_from_organ_data", "path": "aggregator/static_html_crawler.py", "line_start": 2303, "line_end": 2316, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_builder_name", "_clean_text" ], "literal_get_calls": [ { "line": 2309, "receiver": "item", "key": "builder", "expression": "item.get('builder', '')" }, { "line": 2310, "receiver": "item", "key": "date", "expression": "item.get('date', '')" }, { "line": 2311, "receiver": "item", "key": "work", "expression": "item.get('work', '')" }, { "line": 2312, "receiver": "item", "key": "org_nr", "expression": "item.get('org_nr', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2312, "key": "type", "value_expression": "work" }, { "line": 2312, "key": "dates", "value_expression": "[date] if date else []" }, { "line": 2312, "key": "source_id", "value_expression": "str(item.get('org_nr', ''))" }, { "line": 2314, "key": "name", "value_expression": "builder" }, { "line": 2314, "key": "activities", "value_expression": "[work]" } ], "subscript_assignments": [ { "line": 2314, "value_line": 2314, "key": "involved", "target_expression": "event['involved']", "value_expression": "[{'name': builder, 'activities': [work]}]" } ] }, { "symbol": "StaticHtmlOrganCrawler._extract_images", "path": "aggregator/static_html_crawler.py", "line_start": 738, "line_end": 749, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 741, "receiver": "img", "key": "src", "expression": "img.get('src')" }, { "line": 741, "receiver": "img", "key": "data-src", "expression": "img.get('data-src')" }, { "line": 744, "receiver": "img", "key": "alt", "expression": "img.get('alt')" }, { "line": 744, "receiver": "img", "key": "title", "expression": "img.get('title')" } ], "html_queries": [ { "line": 740, "expression": "soup.find_all('img')" } ], "dictionary_entries": [ { "line": 748, "key": "link", "value_expression": "urljoin(page_url or self.base_url, src)" }, { "line": 748, "key": "caption", "value_expression": "alt" }, { "line": 748, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_links", "path": "aggregator/static_html_crawler.py", "line_start": 751, "line_end": 769, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 755, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 764, "key": "url", "value_expression": "full" }, { "line": 765, "key": "label", "value_expression": "label" }, { "line": 766, "key": "scope", "value_expression": "'internal' if parsed.netloc == base_netloc else 'external'" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 538, "line_end": 553, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label", "_clean_text", "_metadata_key" ], "literal_get_calls": [], "html_queries": [ { "line": 540, "expression": "soup.find_all('tr')" }, { "line": 541, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_raw_spec", "path": "aggregator/static_html_crawler.py", "line_start": 578, "line_end": 594, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_multiline", "_division_starts_line", "_parse_divisions_from_text" ], "literal_get_calls": [], "html_queries": [ { "line": 580, "expression": "soup.find_all('table')" }, { "line": 581, "expression": "table.find('table')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_table_rows", "path": "aggregator/static_html_crawler.py", "line_start": 722, "line_end": 736, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 725, "expression": "soup.find_all('table')" }, { "line": 726, "expression": "table.find_all('tr')" }, { "line": 727, "expression": "row.find_all(['th', 'td'], recursive=False)" } ], "dictionary_entries": [ { "line": 731, "key": "table_index", "value_expression": "table_index" }, { "line": 732, "key": "row_index", "value_expression": "row_index" }, { "line": 733, "key": "cells", "value_expression": "cells" }, { "line": 734, "key": "raw_str", "value_expression": "' | '.join(cells)" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._extract_title", "path": "aggregator/static_html_crawler.py", "line_start": 518, "line_end": 524, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 524, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title')" }, { "line": 524, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" } ], "html_queries": [ { "line": 519, "expression": "soup.find('h1')" }, { "line": 519, "expression": "soup.find('h2')" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._fetch_html", "path": "aggregator/static_html_crawler.py", "line_start": 131, "line_end": 145, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_best_decoding", "_declared_html_encoding", "_repair_mojibake" ], "literal_get_calls": [ { "line": 132, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._fetch_json", "path": "aggregator/static_html_crawler.py", "line_start": 2148, "line_end": 2151, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [ { "line": 2149, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._fetch_place", "path": "aggregator/static_html_crawler.py", "line_start": 2153, "line_end": 2155, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_json" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 2154, "key": "id", "value_expression": "place_id" }, { "line": 2154, "key": "lang", "value_expression": "language" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._first_metadata", "path": "aggregator/static_html_crawler.py", "line_start": 825, "line_end": 829, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._identifier_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 268, "line_end": 277, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_detail_url", "path": "aggregator/static_html_crawler.py", "line_start": 213, "line_end": 219, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_division_heading", "path": "aggregator/static_html_crawler.py", "line_start": 681, "line_end": 694, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._is_pitch_fragment", "path": "aggregator/static_html_crawler.py", "line_start": 669, "line_end": 678, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._label_value", "path": "aggregator/static_html_crawler.py", "line_start": 2287, "line_end": 2291, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._location_parts", "path": "aggregator/static_html_crawler.py", "line_start": 569, "line_end": 576, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_split_title" ], "literal_get_calls": [ { "line": 570, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 570, "receiver": "metadata", "key": "town", "expression": "metadata.get('town')" }, { "line": 570, "receiver": "metadata", "key": "location", "expression": "metadata.get('location')" }, { "line": 571, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 571, "receiver": "metadata", "key": "building", "expression": "metadata.get('building')" }, { "line": 571, "receiver": "metadata", "key": "church", "expression": "metadata.get('church')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._looks_like_stop_line", "path": "aggregator/static_html_crawler.py", "line_start": 705, "line_end": 715, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._main_text", "path": "aggregator/static_html_crawler.py", "line_start": 526, "line_end": 536, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text" ], "literal_get_calls": [], "html_queries": [ { "line": 528, "expression": "main.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside'])" }, { "line": 530, "expression": "main.find_all('br')" }, { "line": 532, "expression": "main.find_all(['p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'section'])" }, { "line": 527, "expression": "soup.find('main')" }, { "line": 527, "expression": "soup.find('article')" }, { "line": 527, "expression": "soup.find(id='content')" }, { "line": 527, "expression": "soup.find(class_=re.compile('entry-content|post-content|content', re.I))" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._metadata_key", "path": "aggregator/static_html_crawler.py", "line_start": 555, "line_end": 567, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_label" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_divisions_from_text", "path": "aggregator/static_html_crawler.py", "line_start": 596, "line_end": 666, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_is_division_heading", "_is_pitch_fragment", "_looks_like_stop_line" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._parse_index_page", "path": "aggregator/static_html_crawler.py", "line_start": 202, "line_end": 211, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_clean_text", "_dedupe_refs", "_is_detail_url", "_reference_from_parts", "_title_from_url" ], "literal_get_calls": [], "html_queries": [ { "line": 205, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._preserve_reference_context", "path": "aggregator/static_html_crawler.py", "line_start": 415, "line_end": 432, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_safe_json_loads" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 420, "key": "mapped_extension", "value_expression": "native" }, { "line": 424, "key": "mapped_context", "value_expression": "context" } ], "subscript_assignments": [ { "line": 425, "value_line": 425, "key": "reference_context", "target_expression": "native['reference_context']", "value_expression": "context" }, { "line": 430, "value_line": 430, "key": "raw_data", "target_expression": "context['raw_data']", "value_expression": "self._safe_json_loads(value, value)" } ] }, { "symbol": "StaticHtmlOrganCrawler._reference_from_parts", "path": "aggregator/static_html_crawler.py", "line_start": 221, "line_end": 254, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_identifier_from_url", "_split_title" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 240, "key": "title", "value_expression": "title" }, { "line": 241, "key": "country", "value_expression": "country or self.default_country" }, { "line": 242, "key": "region", "value_expression": "region or self.default_region" }, { "line": 243, "key": "town", "value_expression": "town or parsed_town" }, { "line": 244, "key": "building", "value_expression": "building or parsed_building or title" }, { "line": 245, "key": "builder", "value_expression": "builder" }, { "line": 246, "key": "year", "value_expression": "year" }, { "line": 247, "key": "manual_count", "value_expression": "manual_count" }, { "line": 248, "key": "stop_count", "value_expression": "stop_count" }, { "line": 249, "key": "rank_count", "value_expression": "rank_count" }, { "line": 250, "key": "url", "value_expression": "url" }, { "line": 251, "key": "status", "value_expression": "'initiated'" }, { "line": 252, "key": "raw_data_json", "value_expression": "json.dumps(raw_data or {}, ensure_ascii=False)" }, { "line": 253, "key": "scraped_at", "value_expression": "datetime.now().isoformat()" } ], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._references_from_place", "path": "aggregator/static_html_crawler.py", "line_start": 2157, "line_end": 2200, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_builder_name", "_clean_text", "_label_value", "_reference_from_parts", "_split_sonora_place" ], "literal_get_calls": [ { "line": 2163, "receiver": "place_data", "key": "Ort", "expression": "place_data.get('Ort')" }, { "line": 2164, "receiver": "place_data", "key": "Byggnadens_namn", "expression": "place_data.get('Byggnadens_namn')" }, { "line": 2165, "receiver": "place_data", "key": "Stift", "expression": "place_data.get('Stift')" }, { "line": 2167, "receiver": "(feature or {}).get('properties') or {}", "key": "Building", "expression": "((feature or {}).get('properties') or {}).get('Building', '')" }, { "line": 2172, "receiver": "feature or {}", "key": "geometry", "expression": "(feature or {}).get('geometry')" }, { "line": 2173, "receiver": "geometry", "key": "coordinates", "expression": "geometry.get('coordinates')" }, { "line": 2167, "receiver": "feature or {}", "key": "properties", "expression": "(feature or {}).get('properties')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2192, "key": "place_id", "value_expression": "place_id" }, { "line": 2193, "key": "place_data", "value_expression": "place_data" }, { "line": 2194, "key": "coordinates", "value_expression": "coordinates" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._safe_json_loads", "path": "aggregator/static_html_crawler.py", "line_start": 858, "line_end": 864, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._split_title", "path": "aggregator/static_html_crawler.py", "line_start": 813, "line_end": 822, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonoraCrawler._technical_rows", "path": "aggregator/static_html_crawler.py", "line_start": 2400, "line_end": 2412, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_label_value" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 2411, "key": "label", "value_expression": "key.replace('_', ' ')" }, { "line": 2411, "key": "value", "value_expression": "value" } ], "subscript_assignments": [] }, { "symbol": "StaticHtmlOrganCrawler._title_from_url", "path": "aggregator/static_html_crawler.py", "line_start": 280, "line_end": 283, "inherited": true, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonoraCrawler.crawl", "path": "aggregator/static_html_crawler.py", "line_start": 2091, "line_end": 2146, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_fetch_json", "_fetch_place", "_optional_int", "_references_from_place", "_save_reference", "_update_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 2094, "receiver": "kwargs", "key": "search", "expression": "kwargs.get('search', '')" }, { "line": 2092, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 2093, "receiver": "kwargs", "key": "max_items", "expression": "kwargs.get('max_items')" }, { "line": 2095, "receiver": "kwargs", "key": "building_type", "expression": "kwargs.get('building_type')" }, { "line": 2095, "receiver": "kwargs", "key": "btype", "expression": "kwargs.get('btype')" }, { "line": 2095, "receiver": "self.config", "key": "building_type", "expression": "self.config.get('building_type', 0)" }, { "line": 2096, "receiver": "kwargs", "key": "year1", "expression": "kwargs.get('year1')" }, { "line": 2096, "receiver": "self.config", "key": "year1", "expression": "self.config.get('year1', 1500)" }, { "line": 2097, "receiver": "kwargs", "key": "year2", "expression": "kwargs.get('year2')" }, { "line": 2097, "receiver": "self.config", "key": "year2", "expression": "self.config.get('year2', 1899)" }, { "line": 2098, "receiver": "kwargs", "key": "lang", "expression": "kwargs.get('lang')" }, { "line": 2098, "receiver": "self.config", "key": "lang", "expression": "self.config.get('lang', 'sv')" }, { "line": 2107, "receiver": "index", "key": "features", "expression": "index.get('features', [])" }, { "line": 2112, "receiver": "feature.get('properties') or {}", "key": "Nr", "expression": "(feature.get('properties') or {}).get('Nr')" }, { "line": 2135, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 0.1)" }, { "line": 2112, "receiver": "feature", "key": "properties", "expression": "feature.get('properties')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2105, "key": "seastr", "value_expression": "search" }, { "line": 2105, "key": "btype", "value_expression": "building_type" }, { "line": 2105, "key": "year1", "value_expression": "year1" }, { "line": 2105, "key": "year2", "value_expression": "year2" }, { "line": 2105, "key": "lang", "value_expression": "language" } ], "subscript_assignments": [] }, { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line_start": 2202, "line_end": 2275, "inherited": false, "file_sha256": "f47fb9679d745a7731ee53a7983377a37b0092f7dea66a3500e83cd5181419aa", "self_calls": [ "_builder_name", "_clean_text", "_disposition_from_organ_data", "_documents_from_organ_data", "_events_from_organ_data", "_json_from_raw", "_label_value", "_manual_count", "_place_administrative_fields", "_split_sonora_place", "_technical_rows" ], "literal_get_calls": [ { "line": 2205, "receiver": "cache_data", "key": "place_data", "expression": "cache_data.get('place_data')" }, { "line": 2206, "receiver": "cache_data", "key": "coordinates", "expression": "cache_data.get('coordinates')" }, { "line": 2208, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 2210, "receiver": "raw_data", "key": "town", "expression": "raw_data.get('town')" }, { "line": 2211, "receiver": "raw_data", "key": "building", "expression": "raw_data.get('building')" }, { "line": 2212, "receiver": "raw_data", "key": "region", "expression": "raw_data.get('region')" }, { "line": 2203, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 2203, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 2204, "receiver": "raw_data", "key": "raw_data_json", "expression": "raw_data.get('raw_data_json')" }, { "line": 2205, "receiver": "cache_data", "key": "place_data", "expression": "cache_data.get('place_data')" }, { "line": 2206, "receiver": "cache_data", "key": "coordinates", "expression": "cache_data.get('coordinates')" }, { "line": 2208, "receiver": "organ_data", "key": "Plats", "expression": "organ_data.get('Plats')" }, { "line": 2210, "receiver": "place_data", "key": "Ort", "expression": "place_data.get('Ort')" }, { "line": 2211, "receiver": "place_data", "key": "Byggnadens_namn", "expression": "place_data.get('Byggnadens_namn')" }, { "line": 2212, "receiver": "place_data", "key": "Stift", "expression": "place_data.get('Stift')" }, { "line": 2216, "receiver": "raw_data", "key": "builder", "expression": "raw_data.get('builder', '')" }, { "line": 2217, "receiver": "raw_data", "key": "year", "expression": "raw_data.get('year', '')" }, { "line": 2253, "receiver": "raw_data", "key": "title", "expression": "raw_data.get('title', '')" }, { "line": 2272, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 2271, "receiver": "raw_data", "key": "sonora_id", "expression": "raw_data.get('sonora_id')" }, { "line": 2271, "receiver": "organ_data", "key": "org_id", "expression": "organ_data.get('org_id')" }, { "line": 2251, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 2254, "receiver": "organ_data", "key": "place_id", "expression": "organ_data.get('place_id')" }, { "line": 2254, "receiver": "cache_data", "key": "place_id", "expression": "cache_data.get('place_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 2226, "key": "status", "value_expression": "'current_location'" }, { "line": 2227, "key": "country", "value_expression": "self.default_country" }, { "line": 2228, "key": "region", "value_expression": "region" }, { "line": 2229, "key": "town", "value_expression": "town" }, { "line": 2230, "key": "building", "value_expression": "building" }, { "line": 2231, "key": "raw_str", "value_expression": "place_text" }, { "line": 2238, "key": "raw_specification", "value_expression": "raw_spec" }, { "line": 2239, "key": "stops", "value_expression": "stops_by_division" }, { "line": 2240, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "line": 2241, "key": "manual_count", "value_expression": "self._manual_count(stops_by_division)" }, { "line": 2242, "key": "stop_count", "value_expression": "sum((len(stops) for stops in stops_by_division.values()))" }, { "line": 2243, "key": "technical_rows_unstructured", "value_expression": "self._technical_rows(organ_data)" }, { "line": 2247, "key": "locations", "value_expression": "[location]" }, { "line": 2248, "key": "events", "value_expression": "events" }, { "line": 2249, "key": "specifications", "value_expression": "specifications" }, { "line": 2250, "key": "media", "value_expression": "{'documents': self._documents_from_organ_data(organ_data)}" }, { "line": 2251, "key": "links", "value_expression": "[{'link': raw_data.get('url') or f'{self.api_base_url}/organ1.php?id={raw_data.get(self.id_column)}&lang=sv'}]" }, { "line": 2252, "key": "source_listing", "value_expression": "{'title': raw_data.get('title', ''), 'place_id': str(organ_data.get('place_id') or cache_data.get('place_id') or '')}" }, { "line": 2256, "key": "extensions", "value_expression": "{self.source_key: {'place_data': place_data, 'coordinates': coordinates or []}}" }, { "line": 2265, "key": "organ_data", "value_expression": "organ_data" }, { "line": 2266, "key": "place_context", "value_expression": "cache_data" }, { "line": 2250, "key": "documents", "value_expression": "self._documents_from_organ_data(organ_data)" }, { "line": 2253, "key": "title", "value_expression": "raw_data.get('title', '')" }, { "line": 2254, "key": "place_id", "value_expression": "str(organ_data.get('place_id') or cache_data.get('place_id') or '')" }, { "line": 2219, "key": "type", "value_expression": "'source_reference'" }, { "line": 2219, "key": "dates", "value_expression": "[year] if year else []" }, { "line": 2251, "key": "link", "value_expression": "raw_data.get('url') or f'{self.api_base_url}/organ1.php?id={raw_data.get(self.id_column)}&lang=sv'" }, { "line": 2258, "key": "place_data", "value_expression": "place_data" }, { "line": 2259, "key": "coordinates", "value_expression": "coordinates or []" }, { "line": 2221, "key": "name", "value_expression": "builder" }, { "line": 2221, "key": "activities", "value_expression": "['source_reference']" } ], "subscript_assignments": [ { "line": 2234, "value_line": 2234, "key": "longitude", "target_expression": "location['longitude']", "value_expression": "coordinates[0]" }, { "line": 2235, "value_line": 2235, "key": "latitude", "target_expression": "location['latitude']", "value_expression": "coordinates[1]" }, { "line": 2221, "value_line": 2221, "key": "involved", "target_expression": "event['involved']", "value_expression": "[{'name': builder, 'activities': ['source_reference']}]" } ] } ], "selected_rule_rows": [ { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2242, "key": "stop_count", "value_expression": "sum((len(stops) for stops in stops_by_division.values()))" }, { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2241, "key": "manual_count", "value_expression": "self._manual_count(stops_by_division)" }, { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2240, "key": "stops_by_division", "value_expression": "stops_by_division" }, { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2238, "key": "raw_specification", "value_expression": "raw_spec" }, { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2219, "key": "dates", "value_expression": "[year] if year else []" }, { "symbol": "SonoraCrawler.transform", "path": "aggregator/static_html_crawler.py", "line": 2227, "key": "country", "value_expression": "self.default_country" } ], "bounded_code_passages": [ { "path": "aggregator/static_html_crawler.py", "symbol": "SonoraCrawler.transform", "line_start": 2239, "line_end": 2247, "text": " \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"manual_count\": self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"technical_rows_unstructured\": self._technical_rows(organ_data),\n }\n\n data = {\n \"locations\": [location]," }, { "path": "aggregator/static_html_crawler.py", "symbol": "SonoraCrawler.transform", "line_start": 2238, "line_end": 2246, "text": " \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"manual_count\": self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"technical_rows_unstructured\": self._technical_rows(organ_data),\n }\n\n data = {" }, { "path": "aggregator/static_html_crawler.py", "symbol": "SonoraCrawler.transform", "line_start": 2237, "line_end": 2245, "text": " specifications = {\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"manual_count\": self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"technical_rows_unstructured\": self._technical_rows(organ_data),\n }\n" }, { "path": "aggregator/static_html_crawler.py", "symbol": "SonoraCrawler.transform", "line_start": 2235, "line_end": 2243, "text": " location[\"latitude\"] = coordinates[1]\n\n specifications = {\n \"raw_specification\": raw_spec,\n \"stops\": stops_by_division,\n \"stops_by_division\": stops_by_division,\n \"manual_count\": self._manual_count(stops_by_division),\n \"stop_count\": sum(len(stops) for stops in stops_by_division.values()),\n \"technical_rows_unstructured\": self._technical_rows(organ_data)," }, { "path": "aggregator/static_html_crawler.py", "symbol": "SonoraCrawler.transform", "line_start": 2216, "line_end": 2224, "text": " builder = self._builder_name(raw_data.get(\"builder\", \"\"))\n year = self._clean_text(raw_data.get(\"year\", \"\"))\n if builder or year:\n event = {\"type\": \"source_reference\", \"dates\": [year] if year else []}\n if builder:\n event[\"involved\"] = [{\"name\": builder, \"activities\": [\"source_reference\"]}]\n events.append(event)\n raw_spec, stops_by_division = self._disposition_from_organ_data(organ_data)\n" }, { "path": "aggregator/static_html_crawler.py", "symbol": "SonoraCrawler.transform", "line_start": 2224, "line_end": 2232, "text": "\n location = {\n \"status\": \"current_location\",\n \"country\": self.default_country,\n \"region\": region,\n \"town\": town,\n \"building\": building,\n \"raw_str\": place_text,\n }" } ] } }, { "catalogue_id": "S048", "source_key": "sonus_paradisi_free_stuff", "name": "Sonus Paradisi Free and Demo Sample Sets", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Sonus Paradisi", "scope_description": "Selected subset of a wider commercial organ catalogue", "documentary_unit": "Free or demonstration product announcement", "source_description": "Aggregator targets the producer's Free Stuff blog category rather than the entire shop. The page sits within a catalogue organised by country and instrument, including full products outside the targeted category.", "source_description_references": [ "sonus_paradisi_free_stuff" ], "interpretation": "Navigation links must not be counted as products in the selected collection. A demonstration's stop selection and release date belong to that edition, and should not be inherited from the corresponding full sample set.", "rights_observation": "The category name establishes neither a standard open licence nor unrestricted redistribution of samples.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "sonus_paradisi_free_stuff" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects second-level headings and their associated content. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is free or demonstration product sections. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A demonstration may omit ranks, perspectives or other material from a commercial edition. Demo and full-product specifications cannot be interchanged. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "SonusParadisiFreeStuffCrawler", "class_line": 1332, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "SonusParadisiFreeStuffCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "sonus_paradisi_free_stuff", "path": "aggregator/grandorgue_library_crawler.py", "line": 1333, "class": "SonusParadisiFreeStuffCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "free_demo_catalog", "path": "aggregator/grandorgue_library_crawler.py", "line": 1335, "class": "SonusParadisiFreeStuffCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1337, "line_end": 1396, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1353, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 1353, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" } ], "html_queries": [ { "line": 1340, "expression": "soup.select('h2')" } ], "dictionary_entries": [ { "line": 1360, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Free Stuff'}" }, { "line": 1361, "key": "item", "value_expression": "{'title': title, 'producer_label': 'Sonus Paradisi', 'description_excerpt': _excerpt(text), 'software_labels': ['Hauptwerk'], 'engine_labels': ['Hauptwerk'], 'access_model_hint': 'free_demo' if is_demo else 'free_download_or_free_volume', 'availability_status_hint': 'free_or_demo_catalog_entry'}" }, { "line": 1370, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'demo_sample_set' if is_demo else 'sample_set', 'download_url': _first_download_url(links), 'is_odf_only': False, 'requires_sample_set': False, 'is_demo': is_demo}" }, { "line": 1378, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ', 'represented_organ_label': title, 'manual_count': _first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*manual')), 'pedal_present': 'pedal' in lower, 'stop_count': _first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*stops?'))}" }, { "line": 1385, "key": "links", "value_expression": "links" }, { "line": 1386, "key": "extensions", "value_expression": "{'source_fragment_html': str(parent)}" }, { "line": 1360, "key": "position", "value_expression": "position" }, { "line": 1360, "key": "section_label", "value_expression": "'Free Stuff'" }, { "line": 1362, "key": "title", "value_expression": "title" }, { "line": 1363, "key": "producer_label", "value_expression": "'Sonus Paradisi'" }, { "line": 1364, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1365, "key": "software_labels", "value_expression": "['Hauptwerk']" }, { "line": 1366, "key": "engine_labels", "value_expression": "['Hauptwerk']" }, { "line": 1367, "key": "access_model_hint", "value_expression": "'free_demo' if is_demo else 'free_download_or_free_volume'" }, { "line": 1368, "key": "availability_status_hint", "value_expression": "'free_or_demo_catalog_entry'" }, { "line": 1371, "key": "package_title", "value_expression": "title" }, { "line": 1372, "key": "package_kind", "value_expression": "'demo_sample_set' if is_demo else 'sample_set'" }, { "line": 1373, "key": "download_url", "value_expression": "_first_download_url(links)" }, { "line": 1374, "key": "is_odf_only", "value_expression": "False" }, { "line": 1375, "key": "requires_sample_set", "value_expression": "False" }, { "line": 1376, "key": "is_demo", "value_expression": "is_demo" }, { "line": 1379, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ'" }, { "line": 1380, "key": "represented_organ_label", "value_expression": "title" }, { "line": 1381, "key": "manual_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*manual'))" }, { "line": 1382, "key": "pedal_present", "value_expression": "'pedal' in lower" }, { "line": 1383, "key": "stop_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*stops?'))" }, { "line": 1386, "key": "source_fragment_html", "value_expression": "str(parent)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1383, "key": "stop_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*stops?'))" }, { "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1381, "key": "manual_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*manual'))" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1372, "key": "package_kind", "value_expression": "'demo_sample_set' if is_demo else 'sample_set'" }, { "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1375, "key": "requires_sample_set", "value_expression": "False" }, { "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1374, "key": "is_odf_only", "value_expression": "False" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "line_start": 1380, "line_end": 1388, "text": " \"represented_organ_label\": title,\n \"manual_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*manual\")),\n \"pedal_present\": \"pedal\" in lower,\n \"stop_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*stops?\")),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(parent)},\n },\n primary_id=slugify(detail_url or title)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "line_start": 1378, "line_end": 1386, "text": " \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title,\n \"manual_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*manual\")),\n \"pedal_present\": \"pedal\" in lower,\n \"stop_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*stops?\")),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(parent)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "line_start": 1369, "line_end": 1377, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"demo_sample_set\" if is_demo else \"sample_set\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": is_demo,\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "line_start": 1372, "line_end": 1380, "text": " \"package_kind\": \"demo_sample_set\" if is_demo else \"sample_set\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": is_demo,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\",\n \"represented_organ_label\": title," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SonusParadisiFreeStuffCrawler.parse_html", "line_start": 1371, "line_end": 1379, "text": " \"package_title\": title,\n \"package_kind\": \"demo_sample_set\" if is_demo else \"sample_set\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": False,\n \"requires_sample_set\": False,\n \"is_demo\": is_demo,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\"," } ] } }, { "catalogue_id": "S049", "source_key": "sourceforge_ourorgan_samplesets", "name": "SourceForge GrandOrgue Sample Sets", "selection_role": "registered_source", "parent_source_key": null, "attribution": "GrandOrgue project contributors", "scope_description": "Former GrandOrgue list on SourceForge", "documentary_unit": "Historical sample-set directory entry", "source_description": "The page explicitly says that it is no longer maintained and directs readers to the GrandOrgue GitHub wiki for the current list. It retains free/demo entries and definition files for commercial sets.", "source_description_references": [ "sourceforge_ourorgan_samplesets" ], "interpretation": "Keep this as a historical version and record its successor relationship. Counting it alongside the GitHub list as an independent current catalogue would duplicate much of the evidence and misstate maintenance status.", "rights_observation": "The page directs readers to individual agreements; the listed sample sets have no common licence established here.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "sourceforge_ourorgan_samplesets" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects headings and nested list nodes in the Markdown content. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is items qualified by surrounding headings and parent-list context. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "Parent notes can qualify several files. A download directory is not a census of recorded physical instruments. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "SourceForgeOurOrganSamplesetsCrawler", "class_line": 962, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "SourceForgeOurOrganSamplesetsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "sourceforge_ourorgan_samplesets", "path": "aggregator/grandorgue_library_crawler.py", "line": 963, "class": "SourceForgeOurOrganSamplesetsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "legacy_registry", "path": "aggregator/grandorgue_library_crawler.py", "line": 965, "class": "SourceForgeOurOrganSamplesetsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 967, "line_end": 1035, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [], "html_queries": [ { "line": 974, "expression": "scope.select('h1, h2, h3, li')" }, { "line": 969, "expression": "soup.select_one('.markdown_content')" }, { "line": 979, "expression": "node.find_parent('li').find('p', recursive=False)" }, { "line": 985, "expression": "node.find('ul')" }, { "line": 985, "expression": "node.find('p', recursive=False)" }, { "line": 980, "expression": "node.find_parent('li').find('p', recursive=False)" } ], "dictionary_entries": [ { "line": 1001, "key": "catalog", "value_expression": "{'section_label': current_section, 'position': position, 'legacy_status': 'not_maintained'}" }, { "line": 1002, "key": "item", "value_expression": "{'title': title, 'provider_label': provider_label, 'description_excerpt': _excerpt(text), 'access_model_hint': 'free_or_demo' if any((term in lower for term in ('free', 'demo'))) else 'public_or_unknown', 'availability_status_hint': 'legacy_registry_entry'}" }, { "line": 1009, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'odf' if is_odf else 'download' if is_download else 'registry_entry', 'download_url': _first_download_url(links), 'is_odf_only': is_odf, 'requires_sample_set': is_odf, 'requires_commercial_sample_set': is_odf and 'commercial' in lower_context}" }, { "line": 1017, "key": "organ_relation", "value_expression": "{'pipe_organ_relation_hint': 'sampled_from_real_organ' if 'organ' in lower or 'church' in lower else 'generic_pipe_organ', 'represented_organ_label': title, 'manual_count': _first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*manual')), 'pedal_present': 'pedal' in lower, 'stop_count': _first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*stops?'))}" }, { "line": 1024, "key": "links", "value_expression": "links" }, { "line": 1025, "key": "extensions", "value_expression": "{'source_fragment_html': str(node)}" }, { "line": 1001, "key": "section_label", "value_expression": "current_section" }, { "line": 1001, "key": "position", "value_expression": "position" }, { "line": 1001, "key": "legacy_status", "value_expression": "'not_maintained'" }, { "line": 1003, "key": "title", "value_expression": "title" }, { "line": 1004, "key": "provider_label", "value_expression": "provider_label" }, { "line": 1005, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1006, "key": "access_model_hint", "value_expression": "'free_or_demo' if any((term in lower for term in ('free', 'demo'))) else 'public_or_unknown'" }, { "line": 1007, "key": "availability_status_hint", "value_expression": "'legacy_registry_entry'" }, { "line": 1010, "key": "package_title", "value_expression": "title" }, { "line": 1011, "key": "package_kind", "value_expression": "'odf' if is_odf else 'download' if is_download else 'registry_entry'" }, { "line": 1012, "key": "download_url", "value_expression": "_first_download_url(links)" }, { "line": 1013, "key": "is_odf_only", "value_expression": "is_odf" }, { "line": 1014, "key": "requires_sample_set", "value_expression": "is_odf" }, { "line": 1015, "key": "requires_commercial_sample_set", "value_expression": "is_odf and 'commercial' in lower_context" }, { "line": 1018, "key": "pipe_organ_relation_hint", "value_expression": "'sampled_from_real_organ' if 'organ' in lower or 'church' in lower else 'generic_pipe_organ'" }, { "line": 1019, "key": "represented_organ_label", "value_expression": "title" }, { "line": 1020, "key": "manual_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*manual'))" }, { "line": 1021, "key": "pedal_present", "value_expression": "'pedal' in lower" }, { "line": 1022, "key": "stop_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*stops?'))" }, { "line": 1025, "key": "source_fragment_html", "value_expression": "str(node)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1022, "key": "stop_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*stops?'))" }, { "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1020, "key": "manual_count", "value_expression": "_first_int(_first_match(text, '(?i)(\\\\d+)\\\\s*manual'))" }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1011, "key": "package_kind", "value_expression": "'odf' if is_odf else 'download' if is_download else 'registry_entry'" }, { "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1014, "key": "requires_sample_set", "value_expression": "is_odf" }, { "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1013, "key": "is_odf_only", "value_expression": "is_odf" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "line_start": 1019, "line_end": 1027, "text": " \"represented_organ_label\": title,\n \"manual_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*manual\")),\n \"pedal_present\": \"pedal\" in lower,\n \"stop_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*stops?\")),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)},\n },\n primary_id=slugify(f\"{current_section}-{provider_label}-{title}\")," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "line_start": 1017, "line_end": 1025, "text": " \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\" if \"organ\" in lower or \"church\" in lower else \"generic_pipe_organ\",\n \"represented_organ_label\": title,\n \"manual_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*manual\")),\n \"pedal_present\": \"pedal\" in lower,\n \"stop_count\": _first_int(_first_match(text, r\"(?i)(\\d+)\\s*stops?\")),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)}," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "line_start": 1008, "line_end": 1016, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"odf\" if is_odf else \"download\" if is_download else \"registry_entry\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf,\n \"requires_sample_set\": is_odf,\n \"requires_commercial_sample_set\": is_odf and \"commercial\" in lower_context,\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "line_start": 1011, "line_end": 1019, "text": " \"package_kind\": \"odf\" if is_odf else \"download\" if is_download else \"registry_entry\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf,\n \"requires_sample_set\": is_odf,\n \"requires_commercial_sample_set\": is_odf and \"commercial\" in lower_context,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\" if \"organ\" in lower or \"church\" in lower else \"generic_pipe_organ\",\n \"represented_organ_label\": title," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "SourceForgeOurOrganSamplesetsCrawler.parse_html", "line_start": 1010, "line_end": 1018, "text": " \"package_title\": title,\n \"package_kind\": \"odf\" if is_odf else \"download\" if is_download else \"registry_entry\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf,\n \"requires_sample_set\": is_odf,\n \"requires_commercial_sample_set\": is_odf and \"commercial\" in lower_context,\n },\n \"organ_relation\": {\n \"pipe_organ_relation_hint\": \"sampled_from_real_organ\" if \"organ\" in lower or \"church\" in lower else \"generic_pipe_organ\"," } ] } }, { "catalogue_id": "S050", "source_key": "varhany", "name": "Varhany", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Štěpán Svoboda", "scope_description": "Czech Republic", "documentary_unit": "Linked organ, builder, specification, photograph or literature record", "source_description": "The author dates the private database to 1998 and its web publication to 2005. The about page describes literature, archival work and instrument inspection as inputs, with checked contributions rather than unrestricted direct editing. Some scanned literature is deliberately non-public.", "source_description_references": [ "varhany", "varhany_about", "record_varhany" ], "interpretation": "Its separation of evidence sources is useful for tracing assertions. The site itself says that current-condition information is indicative and often based on literature. Such information needs its documentary date before use as a present-state observation.", "rights_observation": "Copyright is asserted and some supporting scans are restricted; a general redistribution licence was not established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "varhany", "varhany_about", "record_varhany" ], "observation_date": "2026-09-10", "selected_document": { "reference": "record_varhany", "observation_date": "2026-09-11", "selection": "Inherited dossier example; the Orgelseite list was an additional page reached from its homepage. No representative sample is claimed.", "native_form": "Place-centred database page with event rows and expandable specifications", "repeated_information": "Each dated work row associates a builder, activity, specification/literature flags, cost and status.", "date_and_state_distinctions": "Many historical interventions occur within one place record, including predecessor and replacement instruments.", "interpretive_limit": "Not every event has a stoplist. Event rows, organ identities and specification counts are different units.", "observations": [ "Place and church identify the record.", "Nearby organs are listed with distances and linked places.", "Builders appear in dated work rows.", "Repeated work rows contain dates, activities and occasional costs.", "A historical work row expands to a manual/pedal specification.", "Manual/pedal grouping and compact division/stop counts are supplied.", "Literature links are attached to individual work rows.", "Builders, literature and nearby places have separate links.", "A page-update timestamp is displayed." ], "uninspected_or_unfilled_material": [ "Status codes occur in the work rows, but their meaning was not established here; no condition classification is assigned." ], "narrative": "The selected place record arranges dated interventions in rows, with builders, activities, occasional costs and links to literature. A historical work row expands to a manual-and-pedal specification. Not every work event has a stoplist. Status codes appear, but their meaning was not established in this review; no condition classification is assigned to them." }, "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The crawler works with locality and organ references and retains both lok_id and varhany_id. HTML layout classes distinguish columns in the locality or editorial view; tooltips can supply abbreviated technical meanings.", "transformation": "Location, participants, specification, media, literature and links are extracted separately. The specification routine scans the type/work column for a Roman-manual-count and stop-count pattern, reads action information from the adjoining cell and tooltip, and stops at the first matching count row. If no count pattern is found, it retains a textual specification fallback.", "limitations": "The first count-bearing row is not proof of the latest historical state. The inspected specification routine concentrates on summary counts and action context; a full source stoplist cannot be inferred from those counts. A locality identifier and an organ/work identifier should not be collapsed into one unit." }, "implementation": { "class": "VarhanyCrawler", "class_line": 18, "path": "aggregator/varhany_crawler.py", "inheritance": [ "VarhanyCrawler", "BaseCrawler" ], "configuration": {}, "methods": [ { "symbol": "VarhanyCrawler._extract_involved", "path": "aggregator/varhany_crawler.py", "line_start": 553, "line_end": 588, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_clean_text", "_extract_tooltip", "_get_clean_cell_text" ], "literal_get_calls": [], "html_queries": [ { "line": 556, "expression": "soup.find_all('td', class_=re.compile('modul_dva_(lok|red)'))" }, { "line": 573, "expression": "row.find('td', class_=re.compile('modul_jedna_(lok|red)'))" }, { "line": 577, "expression": "row.find('td', class_=re.compile('modul_tri_(lok|red)'))" } ], "dictionary_entries": [ { "line": 584, "key": "name", "value_expression": "self._clean_text(name)" }, { "line": 585, "key": "year", "value_expression": "year" }, { "line": 586, "key": "type_of_work", "value_expression": "self._clean_text(type_of_work)" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler._extract_links", "path": "aggregator/varhany_crawler.py", "line_start": 695, "line_end": 704, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [], "literal_get_calls": [ { "line": 702, "receiver": "a", "key": "href", "expression": "a.get('href')" } ], "html_queries": [ { "line": 697, "expression": "soup.find('div', class_='okolni_varhany')" }, { "line": 699, "expression": "vicinity.find_all('a')" } ], "dictionary_entries": [ { "line": 701, "key": "title", "value_expression": "a.get_text(strip=True)" }, { "line": 702, "key": "link", "value_expression": "urljoin(self.base_url, a.get('href'))" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler._extract_literature", "path": "aggregator/varhany_crawler.py", "line_start": 680, "line_end": 693, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_extract_tooltip" ], "literal_get_calls": [], "html_queries": [ { "line": 682, "expression": "soup.find('table', class_='lokalita_vypis_varhanaru')" }, { "line": 685, "expression": "table.find_all('tr')" }, { "line": 687, "expression": "row.find_all('td')" } ], "dictionary_entries": [ { "line": 692, "key": "raw_str", "value_expression": "bib" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler._extract_locations", "path": "aggregator/varhany_crawler.py", "line_start": 478, "line_end": 551, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_clean_text", "_get_clean_cell_text" ], "literal_get_calls": [], "html_queries": [ { "line": 481, "expression": "soup.find('td', class_='bunka_prvni_obec')" }, { "line": 483, "expression": "cell.find('span', class_='bunka_prvni_patrocinium')" }, { "line": 526, "expression": "soup.find_all('table', class_=re.compile('tabulka_ch|kostel_header'))" }, { "line": 490, "expression": "cell.find('b')" }, { "line": 528, "expression": "soup.find_all('table')" }, { "line": 531, "expression": "table.find('td', align='center')" } ], "dictionary_entries": [ { "line": 514, "key": "town", "value_expression": "match.group(1).strip()" }, { "line": 515, "key": "region", "value_expression": "match.group(2).strip()" }, { "line": 516, "key": "building", "value_expression": "building_text" }, { "line": 520, "key": "town", "value_expression": "town_text" }, { "line": 521, "key": "building", "value_expression": "building_text" }, { "line": 541, "key": "town", "value_expression": "self._clean_text(match.group(1))" }, { "line": 542, "key": "region", "value_expression": "self._clean_text(match.group(2))" }, { "line": 543, "key": "building", "value_expression": "self._clean_text(building)" }, { "line": 547, "key": "town", "value_expression": "self._clean_text(main_geo)" }, { "line": 548, "key": "building", "value_expression": "self._clean_text(building)" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler._extract_media", "path": "aggregator/varhany_crawler.py", "line_start": 644, "line_end": 678, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_clean_text" ], "literal_get_calls": [ { "line": 652, "receiver": "img", "key": "src", "expression": "img.get('src')" } ], "html_queries": [ { "line": 648, "expression": "soup.find_all('img', src=re.compile('show_picture'))" } ], "dictionary_entries": [ { "line": 645, "key": "images", "value_expression": "[]" }, { "line": 675, "key": "link", "value_expression": "link" }, { "line": 676, "key": "copyright", "value_expression": "self._clean_text(copyright_text)" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler._extract_specifications", "path": "aggregator/varhany_crawler.py", "line_start": 590, "line_end": 642, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_extract_tooltip", "_get_clean_cell_text" ], "literal_get_calls": [ { "line": 621, "receiver": "spec", "key": "traction_key", "expression": "spec.get('traction_key')" }, { "line": 625, "receiver": "spec", "key": "traction_key", "expression": "spec.get('traction_key')" } ], "html_queries": [ { "line": 593, "expression": "soup.find_all('td', class_=re.compile('modul_tri_(lok|red)'))" }, { "line": 613, "expression": "row.find('td', class_=re.compile('modul_ctyri_(lok|red)'))" } ], "dictionary_entries": [], "subscript_assignments": [ { "line": 607, "value_line": 607, "key": "manual_count", "target_expression": "spec['manual_count']", "value_expression": "match.group(1)" }, { "line": 608, "value_line": 608, "key": "stop_count", "target_expression": "spec['stop_count']", "value_expression": "match.group(2)" }, { "line": 639, "value_line": 639, "key": "raw_spec", "target_expression": "spec['raw_spec']", "value_expression": "text" }, { "line": 615, "value_line": 615, "key": "traction_key", "target_expression": "spec['traction_key']", "value_expression": "self._get_clean_cell_text(vzd_cell)" }, { "line": 623, "value_line": 623, "key": "windchest", "target_expression": "spec['windchest']", "value_expression": "parts[1].strip()" }, { "line": 622, "value_line": 622, "key": "traction_key", "target_expression": "spec['traction_key']", "value_expression": "parts[0].strip()" }, { "line": 626, "value_line": 626, "key": "traction_key", "target_expression": "spec['traction_key']", "value_expression": "tooltip" }, { "line": 628, "value_line": 628, "key": "windchest", "target_expression": "spec['windchest']", "value_expression": "tooltip" } ] }, { "symbol": "VarhanyCrawler._extract_tooltip", "path": "aggregator/varhany_crawler.py", "line_start": 706, "line_end": 714, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [], "literal_get_calls": [ { "line": 710, "receiver": "element", "key": "onmouseover", "expression": "element.get('onmouseover', '')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler._map_area_labels", "path": "aggregator/varhany_crawler.py", "line_start": 220, "line_end": 229, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler.crawl", "path": "aggregator/varhany_crawler.py", "line_start": 59, "line_end": 188, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_map_area_labels", "_set_crawl_status", "is_scrape_stop_requested", "mark_scrape_stopped" ], "literal_get_calls": [ { "line": 68, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 69, "receiver": "kwargs", "key": "fetch_details", "expression": "kwargs.get('fetch_details', True)" }, { "line": 75, "receiver": "kwargs", "key": "max_locations", "expression": "kwargs.get('max_locations')" }, { "line": 78, "receiver": "kwargs", "key": "timeout", "expression": "kwargs.get('timeout')" }, { "line": 78, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 76, "receiver": "kwargs", "key": "max_retries", "expression": "kwargs.get('max_retries')" }, { "line": 77, "receiver": "kwargs", "key": "retry_delay", "expression": "kwargs.get('retry_delay')" }, { "line": 89, "receiver": "area", "key": "href", "expression": "area.get('href')" }, { "line": 94, "receiver": "area", "key": "onmouseover", "expression": "area.get('onmouseover')" } ], "html_queries": [ { "line": 88, "expression": "soup.find_all('area', href=True)" } ], "dictionary_entries": [ { "line": 96, "key": "lok_id", "value_expression": "lok_id" }, { "line": 97, "key": "url", "value_expression": "urljoin(self.base_url, f'cardheader.php?lok={lok_id}')" }, { "line": 98, "key": "town", "value_expression": "town" }, { "line": 99, "key": "building", "value_expression": "building" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler.preflight_crawl", "path": "aggregator/varhany_crawler.py", "line_start": 190, "line_end": 197, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [], "literal_get_calls": [ { "line": 192, "receiver": "options", "key": "fetch_details", "expression": "options.get('fetch_details', True)" } ], "html_queries": [], "dictionary_entries": [ { "line": 197, "key": "ok", "value_expression": "True" }, { "line": 197, "key": "message", "value_expression": "'Varhany map catalogue and all location pages will be acquired'" }, { "line": 196, "key": "ok", "value_expression": "False" }, { "line": 196, "key": "message", "value_expression": "'Varhany exhaustive crawl requires fetch_details=true'" } ], "subscript_assignments": [] }, { "symbol": "VarhanyCrawler.transform", "path": "aggregator/varhany_crawler.py", "line_start": 417, "line_end": 452, "inherited": false, "file_sha256": "2aa567bfe00ca08330d348c3d4113fba5a0864a8696abd8be4ff796faca7f799", "self_calls": [ "_extract_involved", "_extract_links", "_extract_literature", "_extract_locations", "_extract_media", "_extract_specifications" ], "literal_get_calls": [ { "line": 421, "receiver": "raw_data", "key": "country", "expression": "raw_data.get('country')" }, { "line": 448, "receiver": "raw_data", "key": "varhany_id", "expression": "raw_data.get('varhany_id')" }, { "line": 449, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 450, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html', '')" } ], "html_queries": [], "dictionary_entries": [ { "line": 426, "key": "varhany_id", "value_expression": "raw_data['varhany_id']" }, { "line": 427, "key": "lok_id", "value_expression": "raw_data['lok_id']" }, { "line": 428, "key": "locations", "value_expression": "locations" }, { "line": 429, "key": "involved", "value_expression": "self._extract_involved(soup)" }, { "line": 430, "key": "specifications", "value_expression": "self._extract_specifications(soup)" }, { "line": 431, "key": "media", "value_expression": "self._extract_media(soup)" }, { "line": 432, "key": "literature", "value_expression": "self._extract_literature(soup)" }, { "line": 433, "key": "links", "value_expression": "self._extract_links(soup)" }, { "line": 434, "key": "extensions", "value_expression": "{'varhany': {'reference_context': {key: raw_data.get(key) for key in ('lok_id', 'country', 'town', 'building', 'url') if raw_data.get(key) not in (None, '')}}}" }, { "line": 435, "key": "varhany", "value_expression": "{'reference_context': {key: raw_data.get(key) for key in ('lok_id', 'country', 'town', 'building', 'url') if raw_data.get(key) not in (None, '')}}" }, { "line": 436, "key": "reference_context", "value_expression": "{key: raw_data.get(key) for key in ('lok_id', 'country', 'town', 'building', 'url') if raw_data.get(key) not in (None, '')}" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "VarhanyCrawler._extract_specifications", "path": "aggregator/varhany_crawler.py", "line": 608, "value_line": 608, "key": "stop_count", "target_expression": "spec['stop_count']", "value_expression": "match.group(2)" }, { "symbol": "VarhanyCrawler._extract_specifications", "path": "aggregator/varhany_crawler.py", "line": 607, "value_line": 607, "key": "manual_count", "target_expression": "spec['manual_count']", "value_expression": "match.group(1)" }, { "symbol": "VarhanyCrawler._extract_literature", "path": "aggregator/varhany_crawler.py", "line": 692, "key": "raw_str", "value_expression": "bib" }, { "symbol": "VarhanyCrawler.transform", "path": "aggregator/varhany_crawler.py", "key": "country", "value_expression": "raw_data.get('country')", "line": 421, "kind": "input read" }, { "symbol": "VarhanyCrawler.transform", "path": "aggregator/varhany_crawler.py", "key": "varhany_id", "value_expression": "raw_data.get('varhany_id')", "line": 448, "kind": "input read" }, { "symbol": "VarhanyCrawler.transform", "path": "aggregator/varhany_crawler.py", "key": "url", "value_expression": "raw_data.get('url')", "line": 449, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/varhany_crawler.py", "symbol": "VarhanyCrawler._extract_specifications", "line_start": 605, "line_end": 613, "text": " match = re.search(r\"([IVX]+)\\s*/\\s*(\\d+)\", text)\n if match:\n spec[\"manual_count\"] = match.group(1)\n spec[\"stop_count\"] = match.group(2)\n \n # Once we found the main spec, look for traction in the same row\n row = cell.find_parent(\"tr\")\n if row:\n vzd_cell = row.find(\"td\", class_=re.compile(r\"modul_ctyri_(lok|red)\"))" }, { "path": "aggregator/varhany_crawler.py", "symbol": "VarhanyCrawler._extract_specifications", "line_start": 604, "line_end": 612, "text": " # Look for Manuals/Stops pattern like \"I/5\" or \"II/15\"\n match = re.search(r\"([IVX]+)\\s*/\\s*(\\d+)\", text)\n if match:\n spec[\"manual_count\"] = match.group(1)\n spec[\"stop_count\"] = match.group(2)\n \n # Once we found the main spec, look for traction in the same row\n row = cell.find_parent(\"tr\")\n if row:" }, { "path": "aggregator/varhany_crawler.py", "symbol": "VarhanyCrawler._extract_literature", "line_start": 689, "line_end": 693, "text": " lit_cell = cols[-1]\n bib = self._extract_tooltip(lit_cell)\n if bib:\n lit.append({\"raw_str\": bib})\n return lit" }, { "path": "aggregator/varhany_crawler.py", "symbol": "VarhanyCrawler.transform", "line_start": 418, "line_end": 426, "text": " \"\"\"Main transformation logic.\"\"\"\n soup = BeautifulSoup(raw_data[\"html\"], \"html.parser\")\n locations = self._extract_locations(soup)\n country = raw_data.get(\"country\") or \"Czech Republic\"\n for location in locations:\n location.setdefault(\"country\", country)\n \n transformed = {\n \"varhany_id\": raw_data[\"varhany_id\"]," }, { "path": "aggregator/varhany_crawler.py", "symbol": "VarhanyCrawler.transform", "line_start": 445, "line_end": 452, "text": " return normalize_transfer_record(\n transformed,\n source_key=\"varhany\",\n primary_id=raw_data.get(\"varhany_id\"),\n source_url=raw_data.get(\"url\"),\n raw_content=raw_data.get(\"html\", \"\"),\n content_type=\"text/html\",\n )" }, { "path": "aggregator/varhany_crawler.py", "symbol": "VarhanyCrawler.transform", "line_start": 446, "line_end": 452, "text": " transformed,\n source_key=\"varhany\",\n primary_id=raw_data.get(\"varhany_id\"),\n source_url=raw_data.get(\"url\"),\n raw_content=raw_data.get(\"html\", \"\"),\n content_type=\"text/html\",\n )" } ] } }, { "catalogue_id": "S051", "source_key": "vi_control", "name": "VI-Control", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Frederick Russ / VI-Control", "scope_description": "Broad composer and music-production community; organ material is a subset", "documentary_unit": "Virtual-instrument discussion post or thread", "source_description": "The introduction names Frederick Russ and describes a community for composers, producers and developers of virtual instruments. Organ discussions are selected from this much broader subject range.", "source_description_references": [ "vi_control" ], "interpretation": "Mentions of organ libraries can reveal terminology and user experience. They cannot establish product specifications without identifying the post, product version and relevant producer evidence. Global traffic statements are not counts of organ-related material.", "rights_observation": "No general licence for redistributing discussions or participant data was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "vi_control" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The adapter parses acquired forum-thread pages and supplies forum, thread and post context to a VMI-specific normalizer.", "transformation": "Unlike the general forum-thread family, the transform iterates posts and returns a vmi_forum_thread_batch of individually normalized post records. Each carries author, links, media and thread context. Candidate product or entity mentions are downstream interpretations and must remain distinguishable from the post wording.", "limitations": "Post count, thread count, product mentions and actual products have different denominators. A recommendation or announcement does not validate a product specification. Quotations, publisher statements and other participants' comments require separate attribution." }, "implementation": { "class": "VIControlForumCrawler", "class_line": 30, "path": "aggregator/vi_control_forum_crawler.py", "inheritance": [ "VIControlForumCrawler", "BaseCrawler" ], "configuration": { "record_label": { "value": "posts", "path": "aggregator/vi_control_forum_crawler.py", "line": 38, "class": "VIControlForumCrawler" }, "transformed_json_id_column": { "value": "post_id", "path": "aggregator/vi_control_forum_crawler.py", "line": 40, "class": "VIControlForumCrawler" }, "batch_id_column": { "value": "thread_id", "path": "aggregator/vi_control_forum_crawler.py", "line": 41, "class": "VIControlForumCrawler" }, "cache_id_column": { "value": "thread_id", "path": "aggregator/vi_control_forum_crawler.py", "line": 42, "class": "VIControlForumCrawler" } }, "methods": [ { "symbol": "VIControlForumCrawler._extract_links_and_media", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 514, "line_end": 552, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [], "literal_get_calls": [ { "line": 520, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 539, "receiver": "image", "key": "src", "expression": "image.get('src')" }, { "line": 546, "receiver": "image", "key": "alt", "expression": "image.get('alt')" }, { "line": 546, "receiver": "image", "key": "title", "expression": "image.get('title')" } ], "html_queries": [ { "line": 519, "expression": "node.select('a[href]')" }, { "line": 538, "expression": "node.select('img[src]')" } ], "dictionary_entries": [ { "line": 527, "key": "url", "value_expression": "absolute" }, { "line": 528, "key": "link_text", "value_expression": "_clean_text(link.get_text(' ', strip=True))" }, { "line": 529, "key": "position", "value_expression": "index" }, { "line": 530, "key": "source_selector", "value_expression": "{'type': 'css', 'selector': f'article[data-content=\"post-{post_id}\"] .bbWrapper a:nth-of-type({index})'}" }, { "line": 531, "key": "type", "value_expression": "'css'" }, { "line": 532, "key": "selector", "value_expression": "f'article[data-content=\"post-{post_id}\"] .bbWrapper a:nth-of-type({index})'" }, { "line": 544, "key": "kind", "value_expression": "'image'" }, { "line": 545, "key": "url", "value_expression": "src" }, { "line": 546, "key": "link_text", "value_expression": "image.get('alt') or image.get('title') or ''" }, { "line": 547, "key": "file_name", "value_expression": "os.path.basename(urlparse(src).path)" }, { "line": 548, "key": "position", "value_expression": "index" }, { "line": 549, "key": "source_selector", "value_expression": "{'type': 'css', 'selector': f'article[data-content=\"post-{post_id}\"] img:nth-of-type({index})'}" }, { "line": 537, "key": "kind", "value_expression": "_media_kind(absolute)" }, { "line": 537, "key": "file_name", "value_expression": "os.path.basename(urlparse(absolute).path)" }, { "line": 549, "key": "type", "value_expression": "'css'" }, { "line": 549, "key": "selector", "value_expression": "f'article[data-content=\"post-{post_id}\"] img:nth-of-type({index})'" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler._extract_quotes", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 555, "line_end": 571, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [], "literal_get_calls": [ { "line": 560, "receiver": "quote", "key": "data-source", "expression": "quote.get('data-source')" }, { "line": 562, "receiver": "quote", "key": "data-attributes", "expression": "quote.get('data-attributes')" } ], "html_queries": [ { "line": 559, "expression": "node.select('blockquote')" } ], "dictionary_entries": [ { "line": 565, "key": "type", "value_expression": "'quote'" }, { "line": 566, "key": "cited_user_id", "value_expression": "member_match.group(1) if member_match else ''" }, { "line": 567, "key": "cited_post_id", "value_expression": "post_match.group(1) if post_match else ''" }, { "line": 568, "key": "cited_content_hash", "value_expression": "_text_hash(quote.get_text(' ', strip=True))" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler._parse_forum_thread_rows", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 442, "line_end": 471, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [], "literal_get_calls": [ { "line": 448, "receiver": "link", "key": "href", "expression": "link.get('href')" } ], "html_queries": [ { "line": 447, "expression": "soup.select('a[href*=\"/community/threads/\"]')" }, { "line": 464, "expression": "container.select('.pairs dd')" } ], "dictionary_entries": [ { "line": 458, "key": "thread_id", "value_expression": "thread_id" }, { "line": 459, "key": "forum_id", "value_expression": "forum_id" }, { "line": 460, "key": "forum_title", "value_expression": "forum_title" }, { "line": 461, "key": "title", "value_expression": "_clean_text(link.get_text(' ', strip=True))" }, { "line": 462, "key": "author", "value_expression": "_clean_text(_select_text(container, '.structItem-parts .username, .username'))" }, { "line": 463, "key": "reply_count", "value_expression": "_compact_int(_select_text(container, '.pairs--justified dd, .structItem-cell--meta dd'))" }, { "line": 464, "key": "view_count", "value_expression": "_compact_int(' '.join((node.get_text(' ', strip=True) for node in (container.select('.pairs dd') if container else [])))[::-1][::-1])" }, { "line": 465, "key": "last_post_author", "value_expression": "_clean_text(_select_text(container, '.structItem-cell--latest .username'))" }, { "line": 466, "key": "last_post_at_raw", "value_expression": "_select_attr(container, 'time.u-dt', 'datetime')" }, { "line": 467, "key": "url", "value_expression": "url" }, { "line": 468, "key": "raw_index_html", "value_expression": "str(container) if container else str(link)" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler._parse_post", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 473, "line_end": 512, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [ "_extract_links_and_media", "_extract_quotes" ], "literal_get_calls": [ { "line": 485, "receiver": "author_node", "key": "data-user-id", "expression": "author_node.get('data-user-id')" }, { "line": 486, "receiver": "author_node", "key": "href", "expression": "author_node.get('href')" }, { "line": 486, "receiver": "author_node", "key": "href", "expression": "author_node.get('href')" }, { "line": 498, "receiver": "node", "key": "data-user-id", "expression": "node.get('data-user-id')" }, { "line": 497, "receiver": "quote", "key": "cited_post_id", "expression": "quote.get('cited_post_id')" }, { "line": 498, "receiver": "node", "key": "data-user-id", "expression": "node.get('data-user-id')" } ], "html_queries": [ { "line": 475, "expression": "article.select_one('.bbWrapper')" }, { "line": 483, "expression": "article.select_one('.message-userDetails .username, h4.message-name .username, a.username')" }, { "line": 478, "expression": "body_for_text.select('blockquote')" }, { "line": 498, "expression": "article.select('a.username[data-user-id]')" } ], "dictionary_entries": [ { "line": 489, "key": "post_id", "value_expression": "post_id" }, { "line": 490, "key": "post_url", "value_expression": "_post_url(source_url, post_id)" }, { "line": 491, "key": "position_in_thread", "value_expression": "(page_index - 1) * 20 + position" }, { "line": 492, "key": "posted_at", "value_expression": "_select_attr(article, 'time.u-dt', 'datetime')" }, { "line": 493, "key": "edited_at", "value_expression": "_select_attr(article, \"time.u-dt[title*='edited'], time.u-dt[datetime][data-time]\", 'datetime') if 'edited' in article.get_text(' ', strip=True).lower() else ''" }, { "line": 494, "key": "title", "value_expression": "''" }, { "line": 495, "key": "html", "value_expression": "str(article)" }, { "line": 496, "key": "text", "value_expression": "text" }, { "line": 497, "key": "quoted_post_ids", "value_expression": "[quote['cited_post_id'] for quote in quotes if quote.get('cited_post_id')]" }, { "line": 498, "key": "mentions", "value_expression": "[node.get('data-user-id') for node in article.select('a.username[data-user-id]') if node.get('data-user-id')]" }, { "line": 499, "key": "selectors", "value_expression": "{'post': f'article[data-content=\"post-{post_id}\"]', 'content': f'article[data-content=\"post-{post_id}\"] .bbWrapper'}" }, { "line": 503, "key": "author", "value_expression": "{'source_author_id': author_id, 'display_name': author_name, 'profile_url': profile_url, 'role_label': role}" }, { "line": 509, "key": "links", "value_expression": "links" }, { "line": 510, "key": "media", "value_expression": "media" }, { "line": 511, "key": "quotes", "value_expression": "quotes" }, { "line": 500, "key": "post", "value_expression": "f'article[data-content=\"post-{post_id}\"]'" }, { "line": 501, "key": "content", "value_expression": "f'article[data-content=\"post-{post_id}\"] .bbWrapper'" }, { "line": 504, "key": "source_author_id", "value_expression": "author_id" }, { "line": 505, "key": "display_name", "value_expression": "author_name" }, { "line": 506, "key": "profile_url", "value_expression": "profile_url" }, { "line": 507, "key": "role_label", "value_expression": "role" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler.crawl", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 155, "line_end": 281, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [ "_parse_forum_thread_rows", "_shared_db", "_upsert_scrape_status" ], "literal_get_calls": [ { "line": 160, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 161, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 157, "receiver": "kwargs", "key": "max_forum_pages", "expression": "kwargs.get('max_forum_pages')" }, { "line": 157, "receiver": "self.config", "key": "max_forum_pages", "expression": "self.config.get('max_forum_pages', 1)" }, { "line": 158, "receiver": "kwargs", "key": "max_threads", "expression": "kwargs.get('max_threads')" }, { "line": 158, "receiver": "self.config", "key": "max_threads", "expression": "self.config.get('max_threads', 25)" }, { "line": 159, "receiver": "kwargs", "key": "forum_urls", "expression": "kwargs.get('forum_urls')" }, { "line": 159, "receiver": "self.config", "key": "forum_urls", "expression": "self.config.get('forum_urls')" }, { "line": 242, "receiver": "row", "key": "thread_id", "expression": "row.get('thread_id')" }, { "line": 243, "receiver": "row", "key": "forum_id", "expression": "row.get('forum_id')" }, { "line": 244, "receiver": "row", "key": "forum_title", "expression": "row.get('forum_title')" }, { "line": 245, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 246, "receiver": "row", "key": "author", "expression": "row.get('author')" }, { "line": 247, "receiver": "row", "key": "reply_count", "expression": "row.get('reply_count')" }, { "line": 248, "receiver": "row", "key": "view_count", "expression": "row.get('view_count')" }, { "line": 249, "receiver": "row", "key": "last_post_author", "expression": "row.get('last_post_author')" }, { "line": 250, "receiver": "row", "key": "last_post_at_raw", "expression": "row.get('last_post_at_raw')" }, { "line": 251, "receiver": "row", "key": "url", "expression": "row.get('url')" }, { "line": 253, "receiver": "row", "key": "raw_index_html", "expression": "row.get('raw_index_html')" }, { "line": 262, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" }, { "line": 174, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" }, { "line": 201, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 174, "key": "delay_seconds", "value_expression": "float(self.config.get('rate_limit_delay', 1.5) or 1.5)" }, { "line": 175, "key": "forum_engine", "value_expression": "'xenforo'" }, { "line": 175, "key": "crawl_scope", "value_expression": "'public_threads'" }, { "line": 270, "key": "threads_discovered", "value_expression": "discovered" }, { "line": 270, "key": "incomplete_forums", "value_expression": "incomplete_forums" }, { "line": 184, "key": "source", "value_expression": "self.source_key" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler.parse_thread_html", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 385, "line_end": 389, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [ "parse_thread_pages" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 389, "key": "url", "value_expression": "source_url" }, { "line": 389, "key": "html", "value_expression": "html" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler.parse_thread_pages", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 391, "line_end": 440, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [ "_parse_post" ], "literal_get_calls": [ { "line": 393, "receiver": "pages[0]", "key": "url", "expression": "pages[0].get('url')" }, { "line": 394, "receiver": "pages[0]", "key": "html", "expression": "pages[0].get('html')" }, { "line": 403, "receiver": "page", "key": "url", "expression": "page.get('url')" }, { "line": 407, "receiver": "post", "key": "post_id", "expression": "post.get('post_id')" }, { "line": 430, "receiver": "stats", "key": "reply_count", "expression": "stats.get('reply_count')" }, { "line": 431, "receiver": "stats", "key": "view_count", "expression": "stats.get('view_count')" }, { "line": 404, "receiver": "page", "key": "html", "expression": "page.get('html')" }, { "line": 432, "receiver": "posts[0]", "key": "posted_at", "expression": "posts[0].get('posted_at', '')" }, { "line": 433, "receiver": "posts[-1]", "key": "posted_at", "expression": "posts[-1].get('posted_at', '')" }, { "line": 439, "receiver": "page", "key": "url", "expression": "page.get('url')" }, { "line": 439, "receiver": "page", "key": "url", "expression": "page.get('url')" } ], "html_queries": [ { "line": 399, "expression": "first_soup.select('a.tagItem')" }, { "line": 405, "expression": "soup.select('article.message--post')" }, { "line": 434, "expression": "first_soup.select_one('.structItem-status--locked')" }, { "line": 435, "expression": "first_soup.select_one('.structItem-status--sticky')" } ], "dictionary_entries": [ { "line": 411, "key": "forum", "value_expression": "{'forum_platform': 'XenForo', 'locale': 'en', 'public_access': True, 'credential_required': False, 'terms_url': urljoin(self.base_url, '/community/help/terms/'), 'robots_status': 'not_checked', 'category_id': category_id, 'category_label': category_label, 'category_url': _breadcrumb_forum_href(first_soup)}" }, { "line": 422, "key": "thread", "value_expression": "{'thread_id': thread_id, 'thread_url': source_url or first_url, 'canonical_thread_url': source_url or first_url, 'title': title, 'category_id': category_id, 'category_label': category_label, 'tags': tags, 'reply_count': stats.get('reply_count'), 'view_count': stats.get('view_count'), 'created_at': posts[0].get('posted_at', '') if posts else '', 'updated_at': posts[-1].get('posted_at', '') if posts else '', 'is_locked': bool(first_soup.select_one('.structItem-status--locked')), 'is_sticky': bool(first_soup.select_one('.structItem-status--sticky')), 'is_deleted': False}" }, { "line": 438, "key": "posts", "value_expression": "posts" }, { "line": 439, "key": "source_snapshot", "value_expression": "{'page_urls': [page.get('url') for page in pages if page.get('url')]}" }, { "line": 412, "key": "forum_platform", "value_expression": "'XenForo'" }, { "line": 413, "key": "locale", "value_expression": "'en'" }, { "line": 414, "key": "public_access", "value_expression": "True" }, { "line": 415, "key": "credential_required", "value_expression": "False" }, { "line": 416, "key": "terms_url", "value_expression": "urljoin(self.base_url, '/community/help/terms/')" }, { "line": 417, "key": "robots_status", "value_expression": "'not_checked'" }, { "line": 418, "key": "category_id", "value_expression": "category_id" }, { "line": 419, "key": "category_label", "value_expression": "category_label" }, { "line": 420, "key": "category_url", "value_expression": "_breadcrumb_forum_href(first_soup)" }, { "line": 423, "key": "thread_id", "value_expression": "thread_id" }, { "line": 424, "key": "thread_url", "value_expression": "source_url or first_url" }, { "line": 425, "key": "canonical_thread_url", "value_expression": "source_url or first_url" }, { "line": 426, "key": "title", "value_expression": "title" }, { "line": 427, "key": "category_id", "value_expression": "category_id" }, { "line": 428, "key": "category_label", "value_expression": "category_label" }, { "line": 429, "key": "tags", "value_expression": "tags" }, { "line": 430, "key": "reply_count", "value_expression": "stats.get('reply_count')" }, { "line": 431, "key": "view_count", "value_expression": "stats.get('view_count')" }, { "line": 432, "key": "created_at", "value_expression": "posts[0].get('posted_at', '') if posts else ''" }, { "line": 433, "key": "updated_at", "value_expression": "posts[-1].get('posted_at', '') if posts else ''" }, { "line": 434, "key": "is_locked", "value_expression": "bool(first_soup.select_one('.structItem-status--locked'))" }, { "line": 435, "key": "is_sticky", "value_expression": "bool(first_soup.select_one('.structItem-status--sticky'))" }, { "line": 436, "key": "is_deleted", "value_expression": "False" }, { "line": 439, "key": "page_urls", "value_expression": "[page.get('url') for page in pages if page.get('url')]" }, { "line": 392, "key": "url", "value_expression": "source_url" }, { "line": 392, "key": "html", "value_expression": "''" } ], "subscript_assignments": [] }, { "symbol": "VIControlForumCrawler.transform", "path": "aggregator/vi_control_forum_crawler.py", "line_start": 347, "line_end": 383, "inherited": false, "file_sha256": "6bd444406da3d2fbd896b5c3a13a86d6f2e717a2b063d12670fbadb6c2116875", "self_calls": [ "parse_thread_html", "parse_thread_pages" ], "literal_get_calls": [ { "line": 348, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 348, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 352, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 352, "receiver": "parsed['thread']", "key": "thread_url", "expression": "parsed['thread'].get('thread_url')" }, { "line": 354, "receiver": "raw_data", "key": "forum_title", "expression": "raw_data.get('forum_title')" }, { "line": 354, "receiver": "parsed['thread']", "key": "category_label", "expression": "parsed['thread'].get('category_label')" }, { "line": 357, "receiver": "parsed", "key": "posts", "expression": "parsed.get('posts')" }, { "line": 349, "receiver": "raw_data", "key": "pages", "expression": "raw_data.get('pages')" }, { "line": 351, "receiver": "raw_data", "key": "thread_id", "expression": "raw_data.get('thread_id')" }, { "line": 351, "receiver": "parsed['thread']", "key": "thread_id", "expression": "parsed['thread'].get('thread_id')" }, { "line": 353, "receiver": "raw_data", "key": "forum_id", "expression": "raw_data.get('forum_id')" }, { "line": 353, "receiver": "parsed['thread']", "key": "category_id", "expression": "parsed['thread'].get('category_id')" }, { "line": 381, "receiver": "parsed", "key": "thread", "expression": "parsed.get('thread')" }, { "line": 350, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 350, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 359, "receiver": "parsed", "key": "forum", "expression": "parsed.get('forum')" }, { "line": 360, "receiver": "parsed", "key": "thread", "expression": "parsed.get('thread')" }, { "line": 362, "receiver": "post", "key": "author", "expression": "post.get('author')" }, { "line": 363, "receiver": "post", "key": "links", "expression": "post.get('links')" }, { "line": 364, "receiver": "post", "key": "media", "expression": "post.get('media')" }, { "line": 372, "receiver": "post", "key": "post_id", "expression": "post.get('post_id')" }, { "line": 374, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 375, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" }, { "line": 373, "receiver": "post", "key": "html", "expression": "post.get('html')" } ], "html_queries": [], "dictionary_entries": [ { "line": 379, "key": "record_type", "value_expression": "'vmi_forum_thread_batch'" }, { "line": 380, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.base_url, 'type': 'forum'}" }, { "line": 381, "key": "thread", "value_expression": "parsed.get('thread') or {}" }, { "line": 382, "key": "records", "value_expression": "records" }, { "line": 359, "key": "forum", "value_expression": "parsed.get('forum') or {}" }, { "line": 360, "key": "thread", "value_expression": "parsed.get('thread') or {}" }, { "line": 361, "key": "post", "value_expression": "post" }, { "line": 362, "key": "author", "value_expression": "post.get('author') or {}" }, { "line": 363, "key": "links", "value_expression": "post.get('links') or []" }, { "line": 364, "key": "media", "value_expression": "post.get('media') or []" }, { "line": 365, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 366, "key": "quality", "value_expression": "{'thread_pages_crawled': len(pages) if pages else 1}" }, { "line": 380, "key": "key", "value_expression": "self.source_key" }, { "line": 380, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 380, "key": "base_url", "value_expression": "self.base_url" }, { "line": 380, "key": "type", "value_expression": "'forum'" }, { "line": 366, "key": "thread_pages_crawled", "value_expression": "len(pages) if pages else 1" } ], "subscript_assignments": [ { "line": 351, "value_line": 351, "key": "thread_id", "target_expression": "parsed['thread']['thread_id']", "value_expression": "str(raw_data.get('thread_id') or parsed['thread'].get('thread_id') or '')" }, { "line": 352, "value_line": 352, "key": "thread_url", "target_expression": "parsed['thread']['thread_url']", "value_expression": "raw_data.get('url') or parsed['thread'].get('thread_url') or ''" }, { "line": 353, "value_line": 353, "key": "category_id", "target_expression": "parsed['thread']['category_id']", "value_expression": "str(raw_data.get('forum_id') or parsed['thread'].get('category_id') or '')" }, { "line": 354, "value_line": 354, "key": "category_label", "target_expression": "parsed['thread']['category_label']", "value_expression": "raw_data.get('forum_title') or parsed['thread'].get('category_label') or ''" } ] } ], "selected_rule_rows": [ { "symbol": "VIControlForumCrawler.transform", "path": "aggregator/vi_control_forum_crawler.py", "line": 379, "key": "record_type", "value_expression": "'vmi_forum_thread_batch'" }, { "symbol": "VIControlForumCrawler.parse_thread_pages", "path": "aggregator/vi_control_forum_crawler.py", "line": 439, "key": "source_snapshot", "value_expression": "{'page_urls': [page.get('url') for page in pages if page.get('url')]}" }, { "symbol": "VIControlForumCrawler.transform", "path": "aggregator/vi_control_forum_crawler.py", "key": "html", "value_expression": "raw_data.get('html')", "line": 348, "kind": "input read" }, { "symbol": "VIControlForumCrawler.transform", "path": "aggregator/vi_control_forum_crawler.py", "key": "raw_html", "value_expression": "raw_data.get('raw_html')", "line": 348, "kind": "input read" }, { "symbol": "VIControlForumCrawler.transform", "path": "aggregator/vi_control_forum_crawler.py", "key": "url", "value_expression": "raw_data.get('url')", "line": 352, "kind": "input read" }, { "symbol": "VIControlForumCrawler.transform", "path": "aggregator/vi_control_forum_crawler.py", "key": "thread_url", "value_expression": "parsed['thread'].get('thread_url')", "line": 352, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/vi_control_forum_crawler.py", "symbol": "VIControlForumCrawler.transform", "line_start": 376, "line_end": 383, "text": " )\n )\n return {\n \"record_type\": \"vmi_forum_thread_batch\",\n \"source\": {\"key\": self.source_key, \"display_name\": self.source_display_name, \"base_url\": self.base_url, \"type\": \"forum\"},\n \"thread\": parsed.get(\"thread\") or {},\n \"records\": records,\n }" }, { "path": "aggregator/vi_control_forum_crawler.py", "symbol": "VIControlForumCrawler.parse_thread_pages", "line_start": 436, "line_end": 440, "text": " \"is_deleted\": False,\n },\n \"posts\": posts,\n \"source_snapshot\": {\"page_urls\": [page.get(\"url\") for page in pages if page.get(\"url\")]},\n }" }, { "path": "aggregator/vi_control_forum_crawler.py", "symbol": "VIControlForumCrawler.transform", "line_start": 347, "line_end": 353, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n html = raw_data.get(\"html\") or raw_data.get(\"raw_html\") or \"\"\n pages = _normalize_page_list(raw_data.get(\"pages\")) or _deserialize_thread_pages(html)\n parsed = self.parse_thread_pages(pages, raw_data.get(\"url\") or \"\") if pages else self.parse_thread_html(html, raw_data.get(\"url\") or \"\")\n parsed[\"thread\"][\"thread_id\"] = str(raw_data.get(\"thread_id\") or parsed[\"thread\"].get(\"thread_id\") or \"\")\n parsed[\"thread\"][\"thread_url\"] = raw_data.get(\"url\") or parsed[\"thread\"].get(\"thread_url\") or \"\"\n parsed[\"thread\"][\"category_id\"] = str(raw_data.get(\"forum_id\") or parsed[\"thread\"].get(\"category_id\") or \"\")" }, { "path": "aggregator/vi_control_forum_crawler.py", "symbol": "VIControlForumCrawler.transform", "line_start": 349, "line_end": 357, "text": " pages = _normalize_page_list(raw_data.get(\"pages\")) or _deserialize_thread_pages(html)\n parsed = self.parse_thread_pages(pages, raw_data.get(\"url\") or \"\") if pages else self.parse_thread_html(html, raw_data.get(\"url\") or \"\")\n parsed[\"thread\"][\"thread_id\"] = str(raw_data.get(\"thread_id\") or parsed[\"thread\"].get(\"thread_id\") or \"\")\n parsed[\"thread\"][\"thread_url\"] = raw_data.get(\"url\") or parsed[\"thread\"].get(\"thread_url\") or \"\"\n parsed[\"thread\"][\"category_id\"] = str(raw_data.get(\"forum_id\") or parsed[\"thread\"].get(\"category_id\") or \"\")\n parsed[\"thread\"][\"category_label\"] = raw_data.get(\"forum_title\") or parsed[\"thread\"].get(\"category_label\") or \"\"\n retrieved_at = datetime.now().astimezone().isoformat()\n records = []\n for post in parsed.get(\"posts\") or []:" } ] } }, { "catalogue_id": "S052", "source_key": "virtual_pipe_organs_free_producers", "name": "Virtual Pipe Organs Free Sample Sets and Producers", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Virtual Pipe Organs", "scope_description": "Selected free sample sets and discovery resources", "documentary_unit": "Curated recommendation or producer link", "source_description": "The page expressly describes its selection as non-exhaustive and directs readers to favourite sets, producers and other indexes. Some commentary refers to the producer landscape in early 2020.", "source_description_references": [ "virtual_pipe_organs_free_producers", "vpo_about" ], "interpretation": "This is a dated curatorial selection. Its producer totals and availability statements should not be used as current market statistics. It overlaps with the GrandOrgue lists and with separately registered producer catalogues.", "rights_observation": "The page points to third-party products; permissions must be established at those products.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "virtual_pipe_organs_free_producers", "vpo_about" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects list items within the entry-content area. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is producer-directory entries. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A producer link is a discovery lead, not one sample set or one organ. Free access and redistribution rights are separate questions. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "VirtualPipeOrgansFreeProducersCrawler", "class_line": 1097, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "VirtualPipeOrgansFreeProducersCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "virtual_pipe_organs_free_producers", "path": "aggregator/grandorgue_library_crawler.py", "line": 1098, "class": "VirtualPipeOrgansFreeProducersCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "producer_registry", "path": "aggregator/grandorgue_library_crawler.py", "line": 1100, "class": "VirtualPipeOrgansFreeProducersCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1102, "line_end": 1145, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [], "html_queries": [ { "line": 1104, "expression": "soup.select_one('.entry-content')" }, { "line": 1106, "expression": "scope.select('ul > li')" } ], "dictionary_entries": [ { "line": 1119, "key": "catalog", "value_expression": "{'position': position, 'section_label': 'Free Sample Sets and Producers'}" }, { "line": 1120, "key": "item", "value_expression": "{'title': title, 'provider_label': title, 'description_excerpt': _excerpt(text), 'access_model_hint': 'free_or_demo' if any((term in lower for term in ('free', 'demo', 'shareware'))) else 'public_or_unknown', 'availability_status_hint': 'producer_or_resource_listing'}" }, { "line": 1127, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'odf_reference' if is_odf else 'producer_registry', 'download_url': _first_download_url(links), 'is_odf_only': is_odf and 'sample set' not in lower, 'requires_sample_set': is_odf}" }, { "line": 1134, "key": "links", "value_expression": "links" }, { "line": 1135, "key": "extensions", "value_expression": "{'source_fragment_html': str(node)}" }, { "line": 1119, "key": "position", "value_expression": "position" }, { "line": 1119, "key": "section_label", "value_expression": "'Free Sample Sets and Producers'" }, { "line": 1121, "key": "title", "value_expression": "title" }, { "line": 1122, "key": "provider_label", "value_expression": "title" }, { "line": 1123, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1124, "key": "access_model_hint", "value_expression": "'free_or_demo' if any((term in lower for term in ('free', 'demo', 'shareware'))) else 'public_or_unknown'" }, { "line": 1125, "key": "availability_status_hint", "value_expression": "'producer_or_resource_listing'" }, { "line": 1128, "key": "package_title", "value_expression": "title" }, { "line": 1129, "key": "package_kind", "value_expression": "'odf_reference' if is_odf else 'producer_registry'" }, { "line": 1130, "key": "download_url", "value_expression": "_first_download_url(links)" }, { "line": 1131, "key": "is_odf_only", "value_expression": "is_odf and 'sample set' not in lower" }, { "line": 1132, "key": "requires_sample_set", "value_expression": "is_odf" }, { "line": 1135, "key": "source_fragment_html", "value_expression": "str(node)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1129, "key": "package_kind", "value_expression": "'odf_reference' if is_odf else 'producer_registry'" }, { "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1132, "key": "requires_sample_set", "value_expression": "is_odf" }, { "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1131, "key": "is_odf_only", "value_expression": "is_odf and 'sample set' not in lower" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "line_start": 1126, "line_end": 1134, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"odf_reference\" if is_odf else \"producer_registry\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf and \"sample set\" not in lower,\n \"requires_sample_set\": is_odf,\n },\n \"links\": links," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "line_start": 1129, "line_end": 1137, "text": " \"package_kind\": \"odf_reference\" if is_odf else \"producer_registry\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf and \"sample set\" not in lower,\n \"requires_sample_set\": is_odf,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)},\n },\n primary_id=slugify(title)," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "VirtualPipeOrgansFreeProducersCrawler.parse_html", "line_start": 1128, "line_end": 1136, "text": " \"package_title\": title,\n \"package_kind\": \"odf_reference\" if is_odf else \"producer_registry\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": is_odf and \"sample set\" not in lower,\n \"requires_sample_set\": is_odf,\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(node)},\n }," } ] } }, { "catalogue_id": "S053", "source_key": "virtual_pipe_organs_odfs", "name": "Virtual Pipe Organs ODFs For GrandOrgue", "selection_role": "registered_source", "parent_source_key": null, "attribution": "Virtual Pipe Organs", "scope_description": "GrandOrgue adaptations of selected plain-wave commercial sample sets", "documentary_unit": "Organ definition file for external sample recordings", "source_description": "The page explains that an ODF supplies the structure needed to play compatible recordings in GrandOrgue and links to adaptations for selected commercial sample sets. It distinguishes the definition file from the underlying recording material.", "source_description_references": [ "virtual_pipe_organs_odfs", "vpo_about" ], "interpretation": "Preserve the dependency on the exact external sample set. An ODF is not a complete audio library, and a downloadable ODF cannot be counted as an independently sampled organ.", "rights_observation": "The definition file and recordings require separate permissions; free access to one does not cover the other.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "virtual_pipe_organs_odfs", "vpo_about" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "The source-specific parser selects links within the GrandOrgue ODF section. The library family can enrich discovered items from linked detail pages; catalogue and detail capture are distinct operations.", "transformation": "The implemented unit is definition-file catalogue entries. parse_html produces a collection of item records; the common transform wraps them in a vmi_library_batch. Source fragments, item identity, links and product context are passed to the VMI normalizer. The exact selectors and constructed fields are enumerated in the technical evidence register. The native wording remains necessary to distinguish a producer, a product, a version and the organ represented.", "limitations": "A definition file may require separately obtained samples; it is not by itself a complete recording library. The existence of a parser is not a fresh check of the live catalogue, and a parsed link is not evidence that its downloadable content was inspected." }, "implementation": { "class": "VirtualPipeOrgansOdfsCrawler", "class_line": 1038, "path": "aggregator/grandorgue_library_crawler.py", "inheritance": [ "VirtualPipeOrgansOdfsCrawler", "GrandOrgueLibraryCrawler", "BaseCrawler" ], "configuration": { "source_key": { "value": "virtual_pipe_organs_odfs", "path": "aggregator/grandorgue_library_crawler.py", "line": 1039, "class": "VirtualPipeOrgansOdfsCrawler" }, "catalog_url": { "value": "", "path": "aggregator/grandorgue_library_crawler.py", "line": 34, "class": "GrandOrgueLibraryCrawler" }, "catalog_kind": { "value": "odf_registry", "path": "aggregator/grandorgue_library_crawler.py", "line": 1041, "class": "VirtualPipeOrgansOdfsCrawler" }, "cache_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 40, "class": "GrandOrgueLibraryCrawler" }, "final_source_key_column": { "value": "source_key", "path": "aggregator/grandorgue_library_crawler.py", "line": 41, "class": "GrandOrgueLibraryCrawler" }, "transformed_json_id_column": { "value": "item_id", "path": "aggregator/grandorgue_library_crawler.py", "line": 42, "class": "GrandOrgueLibraryCrawler" } }, "methods": [ { "symbol": "GrandOrgueLibraryCrawler._append_detail_document", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 276, "line_end": 306, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 284, "receiver": "record.get('provenance', {})", "key": "identity", "expression": "record.get('provenance', {}).get('identity', {})" }, { "line": 285, "receiver": "identity", "key": "primary_id_value", "expression": "identity.get('primary_id_value')" }, { "line": 286, "receiver": "record.get('source', {})", "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at')" }, { "line": 284, "receiver": "record", "key": "provenance", "expression": "record.get('provenance', {})" }, { "line": 286, "receiver": "record", "key": "source", "expression": "record.get('source', {})" }, { "line": 300, "receiver": "doc", "key": "document_id", "expression": "doc.get('document_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 288, "key": "document_id", "value_expression": "provenance.document_id(self.source_key, f'{primary_id}:detail', 'raw_html')" }, { "line": 289, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 290, "key": "content_sha256", "value_expression": "provenance.content_sha256(detail_html)" }, { "line": 291, "key": "content_size_bytes", "value_expression": "len(detail_html.encode('utf-8'))" }, { "line": 292, "key": "content_type", "value_expression": "'text/html'" }, { "line": 293, "key": "url", "value_expression": "detail_url" }, { "line": 294, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 295, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 296, "key": "raw_content", "value_expression": "detail_html" } ], "subscript_assignments": [ { "line": 302, "value_line": 302, "key": "detail_page_crawled", "target_expression": "record.setdefault('extensions', {})['detail_page_crawled']", "value_expression": "True" }, { "line": 304, "value_line": 304, "key": "access_model_hint", "target_expression": "record.setdefault('item', {})['access_model_hint']", "value_expression": "'registration_required'" }, { "line": 305, "value_line": 305, "key": "availability_status_hint", "target_expression": "record.setdefault('item', {})['availability_status_hint']", "value_expression": "'available_with_registration'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._apply_detail_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 308, "line_end": 309, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._detail_url_for_record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 259, "line_end": 274, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 261, "receiver": "record", "key": "links", "expression": "record.get('links')" }, { "line": 270, "receiver": "link", "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 272, "receiver": "link", "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 264, "receiver": "link", "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 264, "receiver": "link", "key": "url", "expression": "link.get('url')" }, { "line": 272, "receiver": "link", "key": "rel", "expression": "link.get('rel')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler._enrich_records_with_detail_pages", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 213, "line_end": 257, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_append_detail_document", "_apply_detail_html", "_detail_url_for_record" ], "literal_get_calls": [ { "line": 219, "receiver": "self.config", "key": "crawl_detail_pages", "expression": "self.config.get('crawl_detail_pages')" }, { "line": 225, "receiver": "self.config", "key": "max_detail_pages", "expression": "self.config.get('max_detail_pages', 0)" }, { "line": 243, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 222, "key": "detail_page_enrichment_enabled", "value_expression": "False" }, { "line": 222, "key": "detail_page_fetched", "value_expression": "False" } ], "subscript_assignments": [ { "line": 230, "value_line": 230, "key": "detail_page_enrichment_enabled", "target_expression": "completeness['detail_page_enrichment_enabled']", "value_expression": "True" }, { "line": 255, "value_line": 255, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "True" }, { "line": 232, "value_line": 232, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 233, "value_line": 233, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'configured_detail_page_limit'" }, { "line": 238, "value_line": 238, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 239, "value_line": 239, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'no_detail_url_discovered'" }, { "line": 247, "value_line": 247, "key": "detail_page_fetched", "target_expression": "completeness['detail_page_fetched']", "value_expression": "False" }, { "line": 248, "value_line": 248, "key": "detail_page_omission_reason", "target_expression": "completeness['detail_page_omission_reason']", "value_expression": "'fetch_failed'" } ] }, { "symbol": "GrandOrgueLibraryCrawler._record", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 311, "line_end": 341, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [], "literal_get_calls": [ { "line": 326, "receiver": "payload", "key": "catalog", "expression": "payload.get('catalog')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [ { "line": 330, "value_line": 330, "key": "catalog", "target_expression": "payload['catalog']", "value_expression": "catalog" } ] }, { "symbol": "GrandOrgueLibraryCrawler.crawl", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 86, "line_end": 136, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_enrich_records_with_detail_pages", "_fail_shared_crawl_run", "_finish_shared_crawl_run", "_register_shared_crawl_target", "_upsert_scrape_status", "parse_html" ], "literal_get_calls": [ { "line": 88, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 89, "receiver": "kwargs", "key": "run_id", "expression": "kwargs.get('run_id')" }, { "line": 87, "receiver": "kwargs", "key": "url", "expression": "kwargs.get('url')" }, { "line": 91, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 121, "receiver": "record.get('item', {})", "key": "title", "expression": "record.get('item', {}).get('title')" }, { "line": 122, "receiver": "record.get('source_snapshot', {})", "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 124, "receiver": "record.get('extensions', {})", "key": "source_fragment_html", "expression": "record.get('extensions', {}).get('source_fragment_html')" }, { "line": 121, "receiver": "record", "key": "item", "expression": "record.get('item', {})" }, { "line": 122, "receiver": "record", "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 124, "receiver": "record", "key": "extensions", "expression": "record.get('extensions', {})" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 1043, "line_end": 1094, "inherited": false, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "_record" ], "literal_get_calls": [ { "line": 1051, "receiver": "link", "key": "href", "expression": "link.get('href')" }, { "line": 1062, "receiver": "item", "key": "normalized_url", "expression": "item.get('normalized_url')" } ], "html_queries": [ { "line": 1049, "expression": "soup.select('a[href*=\"/sample-sets/odfs-for-grandorgue/\"]')" } ], "dictionary_entries": [ { "line": 1063, "key": "url", "value_expression": "detail_url" }, { "line": 1063, "key": "link_text", "value_expression": "title" }, { "line": 1063, "key": "is_product_homepage_candidate", "value_expression": "True" }, { "line": 1067, "key": "catalog", "value_expression": "{'section_label': 'ODFs For GrandOrgue', 'position': position}" }, { "line": 1068, "key": "item", "value_expression": "{'title': title, 'provider_label': 'Virtual Pipe Organs', 'description_excerpt': _excerpt(text), 'access_model_hint': 'free_odf_download_or_reference', 'availability_status_hint': 'odf_available_or_linked'}" }, { "line": 1075, "key": "package", "value_expression": "{'package_title': title, 'package_kind': 'odf', 'download_url': _first_download_url(links), 'is_odf_only': True, 'requires_sample_set': True, 'requires_commercial_sample_set': any((term in text.lower() for term in ('commercial', 'purchase', 'sonus paradisi')))}" }, { "line": 1083, "key": "links", "value_expression": "links" }, { "line": 1084, "key": "extensions", "value_expression": "{'source_fragment_html': str(parent)}" }, { "line": 1067, "key": "section_label", "value_expression": "'ODFs For GrandOrgue'" }, { "line": 1067, "key": "position", "value_expression": "position" }, { "line": 1069, "key": "title", "value_expression": "title" }, { "line": 1070, "key": "provider_label", "value_expression": "'Virtual Pipe Organs'" }, { "line": 1071, "key": "description_excerpt", "value_expression": "_excerpt(text)" }, { "line": 1072, "key": "access_model_hint", "value_expression": "'free_odf_download_or_reference'" }, { "line": 1073, "key": "availability_status_hint", "value_expression": "'odf_available_or_linked'" }, { "line": 1076, "key": "package_title", "value_expression": "title" }, { "line": 1077, "key": "package_kind", "value_expression": "'odf'" }, { "line": 1078, "key": "download_url", "value_expression": "_first_download_url(links)" }, { "line": 1079, "key": "is_odf_only", "value_expression": "True" }, { "line": 1080, "key": "requires_sample_set", "value_expression": "True" }, { "line": 1081, "key": "requires_commercial_sample_set", "value_expression": "any((term in text.lower() for term in ('commercial', 'purchase', 'sonus paradisi')))" }, { "line": 1084, "key": "source_fragment_html", "value_expression": "str(parent)" } ], "subscript_assignments": [] }, { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line_start": 138, "line_end": 156, "inherited": true, "file_sha256": "8654fc0b5874b2b520ce43cef5c43fd65c03c97307b1b39fbc1acca68adc4b65", "self_calls": [ "parse_html" ], "literal_get_calls": [ { "line": 139, "receiver": "raw_data", "key": "html", "expression": "raw_data.get('html')" }, { "line": 139, "receiver": "raw_data", "key": "raw_html", "expression": "raw_data.get('raw_html')" }, { "line": 139, "receiver": "raw_data", "key": "markdown", "expression": "raw_data.get('markdown')" }, { "line": 140, "receiver": "raw_data", "key": "url", "expression": "raw_data.get('url')" }, { "line": 140, "receiver": "raw_data", "key": "source_url", "expression": "raw_data.get('source_url')" }, { "line": 144, "receiver": "raw_data", "key": "crawl_run_id", "expression": "raw_data.get('crawl_run_id')" }, { "line": 145, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [ { "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "line": 149, "key": "source", "value_expression": "{'key': self.source_key, 'display_name': self.source_display_name, 'base_url': self.catalog_url, 'type': self.source_type}" }, { "line": 155, "key": "records", "value_expression": "records" }, { "line": 150, "key": "key", "value_expression": "self.source_key" }, { "line": 151, "key": "display_name", "value_expression": "self.source_display_name" }, { "line": 152, "key": "base_url", "value_expression": "self.catalog_url" }, { "line": 153, "key": "type", "value_expression": "self.source_type" } ], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "GrandOrgueLibraryCrawler.transform", "path": "aggregator/grandorgue_library_crawler.py", "line": 148, "key": "record_type", "value_expression": "'vmi_library_batch'" }, { "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1077, "key": "package_kind", "value_expression": "'odf'" }, { "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1080, "key": "requires_sample_set", "value_expression": "True" }, { "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "path": "aggregator/grandorgue_library_crawler.py", "line": 1079, "key": "is_odf_only", "value_expression": "True" } ], "bounded_code_passages": [ { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "GrandOrgueLibraryCrawler.transform", "line_start": 145, "line_end": 153, "text": " transform_run_id=raw_data.get(\"transform_run_id\"),\n )\n return {\n \"record_type\": \"vmi_library_batch\",\n \"source\": {\n \"key\": self.source_key,\n \"display_name\": self.source_display_name,\n \"base_url\": self.catalog_url,\n \"type\": self.source_type," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "line_start": 1074, "line_end": 1082, "text": " },\n \"package\": {\n \"package_title\": title,\n \"package_kind\": \"odf\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": True,\n \"requires_sample_set\": True,\n \"requires_commercial_sample_set\": any(term in text.lower() for term in (\"commercial\", \"purchase\", \"sonus paradisi\")),\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "line_start": 1077, "line_end": 1085, "text": " \"package_kind\": \"odf\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": True,\n \"requires_sample_set\": True,\n \"requires_commercial_sample_set\": any(term in text.lower() for term in (\"commercial\", \"purchase\", \"sonus paradisi\")),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(parent)},\n }," }, { "path": "aggregator/grandorgue_library_crawler.py", "symbol": "VirtualPipeOrgansOdfsCrawler.parse_html", "line_start": 1076, "line_end": 1084, "text": " \"package_title\": title,\n \"package_kind\": \"odf\",\n \"download_url\": _first_download_url(links),\n \"is_odf_only\": True,\n \"requires_sample_set\": True,\n \"requires_commercial_sample_set\": any(term in text.lower() for term in (\"commercial\", \"purchase\", \"sonus paradisi\")),\n },\n \"links\": links,\n \"extensions\": {\"source_fragment_html\": str(parent)}," } ] } }, { "catalogue_id": "S054", "source_key": "score_pdmx", "name": "PDMX Public Domain MusicXML", "selection_role": "score_collection", "parent_source_key": "digital_sheet_music", "attribution": "Phillip Long; Zachary Novack; Julian McAuley; Taylor Berg-Kirkpatrick", "scope_description": "PDMX v9; broad music corpus with organ-relevant subsets", "documentary_unit": "Dataset row and associated symbolic or rendered files", "source_description": "The pinned Zenodo record is version v9, published 1 June 2025. Its description reports 31,221 records with conflicting public and embedded licence metadata and recommends the no_license_conflict subset. It separately identifies files that could not be converted from corrupted originals.", "source_description_references": [ "score_pdmx" ], "interpretation": "Retain the version DOI, row identifier, conflict flag and file format. An organ subset must be defined reproducibly; the full dataset size is not a count of organ scores. The authors request citations to both their paper and their collaborators' paper when the dataset is used.", "rights_observation": "The record carries CC BY 4.0 and reports item-level licence conflicts; its title is not proof that every file is free of conflicting claims.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "score_pdmx" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "This is the pdmx collection route inside DigitalSheetMusicCrawler, not a separately registered crawler. The score-level route reads a supplied local dataset snapshot. A separate Zenodo-inventory route yields dataset/file metadata and an access-blocked status rather than automatically downloading the multi-gigabyte collection.", "transformation": "The shared score normalizer separates work metadata, assets, formats, collection context and licence statements. Dataset-row filtering and asset selection are separate functions. A dataset inventory item must not be counted as one successfully acquired score. Licence-conflict flags, failed conversions and duplicate representations require the dataset's own definitions.", "limitations": "Retain the version DOI, row identifier, conflict flag and file format. An organ subset must be defined reproducibly; the full dataset size is not a count of organ scores. The authors request citations to both their paper and their collaborators' paper when the dataset is used. Availability of a collection route does not demonstrate that all pages or files were acquired." }, "implementation": { "class": "DigitalSheetMusicCrawler", "class_line": 51, "path": "aggregator/digital_sheet_music_crawler.py", "inheritance": [ "DigitalSheetMusicCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "score_id", "path": "aggregator/digital_sheet_music_crawler.py", "line": 58, "class": "DigitalSheetMusicCrawler" }, "record_label": { "value": "digital scores", "path": "aggregator/digital_sheet_music_crawler.py", "line": 59, "class": "DigitalSheetMusicCrawler" } }, "methods": [ { "symbol": "DigitalSheetMusicCrawler._assets_from_html", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 857, "line_end": 905, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 865, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 864, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 887, "key": "url", "value_expression": "href" }, { "line": 887, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 887, "key": "format", "value_expression": "info['format']" }, { "line": 887, "key": "label", "value_expression": "text or info['format']" }, { "line": 897, "key": "url", "value_expression": "href" }, { "line": 898, "key": "alternate_urls", "value_expression": "_imslp_mirror_candidates(href, imslp_file_id)" }, { "line": 899, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 900, "key": "format", "value_expression": "info['format']" }, { "line": 901, "key": "label", "value_expression": "text or info['format']" }, { "line": 902, "key": "source_file_id", "value_expression": "imslp_file_id" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_collection", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 318, "line_end": 329, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_imslp_wima", "_crawl_musescore_direct", "_crawl_mutopia", "_crawl_orgelbuechlein", "_crawl_pdmx" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_imslp_wima", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 457, "line_end": 484, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 459, "receiver": "options", "key": "imslp_seed_urls", "expression": "options.get('imslp_seed_urls')" }, { "line": 460, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 461, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 479, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [], "dictionary_entries": [ { "line": 480, "key": "html_excerpt", "value_expression": "html[:10000]" }, { "line": 480, "key": "seed_url", "value_expression": "url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_musescore_direct", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 513, "line_end": 563, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 514, "receiver": "options", "key": "musescore_direct_urls", "expression": "options.get('musescore_direct_urls')" }, { "line": 515, "receiver": "self.config", "key": "musescore_credentials_env", "expression": "self.config.get('musescore_credentials_env')" }, { "line": 539, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 543, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 532, "key": "credential_required", "value_expression": "True" }, { "line": 533, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "credential_required", "value_expression": "True" }, { "line": 559, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "http_status", "value_expression": "response.status_code" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_mutopia", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 486, "line_end": 511, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_mutopia_next_url", "_parse_mutopia_listing", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 487, "receiver": "self.config", "key": "mutopia_collection_url", "expression": "self.config.get('mutopia_collection_url')" }, { "line": 488, "receiver": "self.config", "key": "mutopia_max_pages", "expression": "self.config.get('mutopia_max_pages', 0)" }, { "line": 496, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 500, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_options", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 280, "line_end": 316, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "source", "key": "collection", "expression": "source.get('collection')" }, { "line": 286, "receiver": "source", "key": "collections", "expression": "source.get('collections')" }, { "line": 293, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 295, "receiver": "source", "key": "organ_only", "expression": "source.get('organ_only')" }, { "line": 302, "receiver": "source", "key": "pdmx_enable_archive_streaming", "expression": "source.get('pdmx_enable_archive_streaming')" }, { "line": 288, "receiver": "self.config", "key": "default_collections", "expression": "self.config.get('default_collections')" }, { "line": 297, "receiver": "source", "key": "pdmx_csv_path", "expression": "source.get('pdmx_csv_path')" }, { "line": 297, "receiver": "self.config", "key": "pdmx_csv_path", "expression": "self.config.get('pdmx_csv_path')" }, { "line": 299, "receiver": "source", "key": "pdmx_root_dir", "expression": "source.get('pdmx_root_dir')" }, { "line": 299, "receiver": "self.config", "key": "pdmx_root_dir", "expression": "self.config.get('pdmx_root_dir')" }, { "line": 300, "receiver": "source", "key": "pdmx_zenodo_record_api_url", "expression": "source.get('pdmx_zenodo_record_api_url')" }, { "line": 300, "receiver": "self.config", "key": "pdmx_zenodo_record_api_url", "expression": "self.config.get('pdmx_zenodo_record_api_url')" }, { "line": 303, "receiver": "self.config", "key": "pdmx_enable_archive_streaming", "expression": "self.config.get('pdmx_enable_archive_streaming', False)" }, { "line": 306, "receiver": "source", "key": "pdmx_archive_stream_max_mb", "expression": "source.get('pdmx_archive_stream_max_mb')" }, { "line": 306, "receiver": "self.config", "key": "pdmx_archive_stream_max_mb", "expression": "self.config.get('pdmx_archive_stream_max_mb')" }, { "line": 314, "receiver": "source", "key": "imslp_seed_urls", "expression": "source.get('imslp_seed_urls')" }, { "line": 315, "receiver": "source", "key": "musescore_direct_urls", "expression": "source.get('musescore_direct_urls')" }, { "line": 310, "receiver": "source", "key": "pdmx_mxl_archive_url", "expression": "source.get('pdmx_mxl_archive_url')" }, { "line": 310, "receiver": "self.config", "key": "pdmx_mxl_archive_url", "expression": "self.config.get('pdmx_mxl_archive_url')" }, { "line": 311, "receiver": "source", "key": "pdmx_mid_archive_url", "expression": "source.get('pdmx_mid_archive_url')" }, { "line": 311, "receiver": "self.config", "key": "pdmx_mid_archive_url", "expression": "self.config.get('pdmx_mid_archive_url')" }, { "line": 312, "receiver": "source", "key": "pdmx_pdf_archive_url", "expression": "source.get('pdmx_pdf_archive_url')" }, { "line": 312, "receiver": "self.config", "key": "pdmx_pdf_archive_url", "expression": "self.config.get('pdmx_pdf_archive_url')" }, { "line": 314, "receiver": "self.config", "key": "imslp_seed_urls", "expression": "self.config.get('imslp_seed_urls')" }, { "line": 315, "receiver": "self.config", "key": "musescore_direct_urls", "expression": "self.config.get('musescore_direct_urls')" }, { "line": 294, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 296, "receiver": "source", "key": "license_filter", "expression": "source.get('license_filter')" }, { "line": 298, "receiver": "source", "key": "pdmx_mode", "expression": "source.get('pdmx_mode')" }, { "line": 298, "receiver": "self.config", "key": "pdmx_mode", "expression": "self.config.get('pdmx_mode')" } ], "html_queries": [], "dictionary_entries": [ { "line": 292, "key": "collections", "value_expression": "[collection for collection in collections if collection in SOURCE_COLLECTIONS]" }, { "line": 293, "key": "limit", "value_expression": "_safe_int(source.get('limit'), default=0)" }, { "line": 294, "key": "query", "value_expression": "str(source.get('query') or '').strip().lower()" }, { "line": 295, "key": "organ_only", "value_expression": "_read_bool(source.get('organ_only'), True)" }, { "line": 296, "key": "license_filter", "value_expression": "str(source.get('license_filter') or '').strip().lower()" }, { "line": 297, "key": "pdmx_csv_path", "value_expression": "str(source.get('pdmx_csv_path') or self.config.get('pdmx_csv_path') or '')" }, { "line": 298, "key": "pdmx_mode", "value_expression": "str(source.get('pdmx_mode') or self.config.get('pdmx_mode') or 'rows').strip().lower()" }, { "line": 299, "key": "pdmx_root_dir", "value_expression": "str(source.get('pdmx_root_dir') or self.config.get('pdmx_root_dir') or '')" }, { "line": 300, "key": "pdmx_zenodo_record_api_url", "value_expression": "str(source.get('pdmx_zenodo_record_api_url') or self.config.get('pdmx_zenodo_record_api_url') or '')" }, { "line": 301, "key": "pdmx_enable_archive_streaming", "value_expression": "_read_bool(source.get('pdmx_enable_archive_streaming'), bool(self.config.get('pdmx_enable_archive_streaming', False)))" }, { "line": 305, "key": "pdmx_archive_stream_max_mb", "value_expression": "_safe_int(source.get('pdmx_archive_stream_max_mb') or self.config.get('pdmx_archive_stream_max_mb'), default=256)" }, { "line": 309, "key": "pdmx_archive_urls", "value_expression": "{'mxl': str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or ''), 'mid': str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or ''), 'pdf': str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')}" }, { "line": 314, "key": "imslp_seed_urls", "value_expression": "_split_lines(source.get('imslp_seed_urls')) or list(self.config.get('imslp_seed_urls') or [])" }, { "line": 315, "key": "musescore_direct_urls", "value_expression": "_split_lines(source.get('musescore_direct_urls')) or list(self.config.get('musescore_direct_urls') or [])" }, { "line": 310, "key": "mxl", "value_expression": "str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or '')" }, { "line": 311, "key": "mid", "value_expression": "str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or '')" }, { "line": 312, "key": "pdf", "value_expression": "str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')" } ], "subscript_assignments": [ { "line": 285, "value_line": 285, "key": "collection", "target_expression": "source['collection']", "value_expression": "str(target)" } ] }, { "symbol": "DigitalSheetMusicCrawler._crawl_orgelbuechlein", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 416, "line_end": 455, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 417, "receiver": "self.config", "key": "orgelbuechlein_api_url", "expression": "self.config.get('orgelbuechlein_api_url')" }, { "line": 429, "receiver": "item", "key": "download_url", "expression": "item.get('download_url')" }, { "line": 429, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 418, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 423, "receiver": "item", "key": "name", "expression": "item.get('name')" }, { "line": 436, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 453, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "github_item", "value_expression": "item" }, { "line": 442, "key": "url", "value_expression": "raw_url" }, { "line": 442, "key": "file_name", "value_expression": "name" }, { "line": 442, "key": "format", "value_expression": "asset_format_from_url(name)['format']" }, { "line": 442, "key": "label", "value_expression": "'MusicXML'" }, { "line": 444, "key": "url", "value_expression": "raw_url.replace('/little-organ-book-mxl/', '/little-organ-book-pdf/').rsplit('.', 1)[0] + '.pdf'" }, { "line": 445, "key": "file_name", "value_expression": "f'{stem}.pdf'" }, { "line": 446, "key": "format", "value_expression": "'pdf'" }, { "line": 447, "key": "label", "value_expression": "'PDF'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 331, "line_end": 364, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_pdmx_zenodo_inventory", "_pdmx_assets", "_pdmx_row_matches", "_raw_record", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 332, "receiver": "options", "key": "pdmx_mode", "expression": "options.get('pdmx_mode')" }, { "line": 338, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 334, "receiver": "options", "key": "pdmx_csv_path", "expression": "options.get('pdmx_csv_path')" }, { "line": 337, "receiver": "options", "key": "pdmx_root_dir", "expression": "options.get('pdmx_root_dir')" }, { "line": 338, "receiver": "self.config", "key": "pdmx_max_rows", "expression": "self.config.get('pdmx_max_rows')" }, { "line": 348, "receiver": "row", "key": "path", "expression": "row.get('path')" }, { "line": 348, "receiver": "row", "key": "mxl", "expression": "row.get('mxl')" }, { "line": 348, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "song_name", "expression": "row.get('song_name')" }, { "line": 354, "receiver": "row", "key": "composer_name", "expression": "row.get('composer_name')" }, { "line": 354, "receiver": "row", "key": "artist_name", "expression": "row.get('artist_name')" }, { "line": 356, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 357, "receiver": "row", "key": "license_url", "expression": "row.get('license_url')" }, { "line": 359, "receiver": "row", "key": "instrumentation", "expression": "row.get('instrumentation')" }, { "line": 359, "receiver": "row", "key": "instruments", "expression": "row.get('instruments')" }, { "line": 359, "receiver": "row", "key": "instrument", "expression": "row.get('instrument')" } ], "html_queries": [], "dictionary_entries": [ { "line": 361, "key": "pdmx", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx_zenodo_inventory", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 366, "line_end": 414, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 373, "receiver": "payload", "key": "files", "expression": "payload.get('files')" }, { "line": 390, "receiver": "payload.get('metadata', {})", "key": "title", "expression": "payload.get('metadata', {}).get('title')" }, { "line": 367, "receiver": "options", "key": "pdmx_zenodo_record_api_url", "expression": "options.get('pdmx_zenodo_record_api_url')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "content", "expression": "item.get('links', {}).get('content')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "self", "expression": "item.get('links', {}).get('self')" }, { "line": 370, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 376, "receiver": "item", "key": "key", "expression": "item.get('key')" }, { "line": 386, "receiver": "item", "key": "size", "expression": "item.get('size')" }, { "line": 390, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 397, "receiver": "payload.get('links', {})", "key": "html", "expression": "payload.get('links', {}).get('html')" }, { "line": 398, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "id", "expression": "payload.get('metadata', {}).get('license', {}).get('id')" }, { "line": 399, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "url", "expression": "payload.get('metadata', {}).get('license', {}).get('url')" }, { "line": 406, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 407, "receiver": "payload", "key": "doi", "expression": "payload.get('doi')" }, { "line": 393, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 397, "receiver": "payload", "key": "links", "expression": "payload.get('links', {})" }, { "line": 398, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 399, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 398, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 399, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 382, "key": "url", "value_expression": "content_url" }, { "line": 383, "key": "file_name", "value_expression": "key" }, { "line": 384, "key": "format", "value_expression": "asset_format_from_url(key)['format']" }, { "line": 385, "key": "label", "value_expression": "'Zenodo dataset file'" }, { "line": 386, "key": "content_size_bytes", "value_expression": "item.get('size')" }, { "line": 387, "key": "download_status", "value_expression": "'dataset_snapshot_required'" }, { "line": 405, "key": "zenodo_record", "value_expression": "{'id': payload.get('id'), 'doi': payload.get('doi'), 'api_url': api_url, 'files': files}" }, { "line": 411, "key": "access_note", "value_expression": "'PDMX requires an explicit local snapshot path for score-level crawling to avoid automatic multi-GB dataset downloads.'" }, { "line": 406, "key": "id", "value_expression": "payload.get('id')" }, { "line": 407, "key": "doi", "value_expression": "payload.get('doi')" }, { "line": 408, "key": "api_url", "value_expression": "api_url" }, { "line": 409, "key": "files", "value_expression": "files" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._extract_tar_gz_member_from_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 780, "line_end": 812, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 785, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._mutopia_next_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 990, "line_end": 995, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 994, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 992, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._parse_mutopia_listing", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 907, "line_end": 988, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 939, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 951, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 912, "receiver": "row", "key": "class", "expression": "row.get('class')" }, { "line": 980, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [ { "line": 911, "expression": "soup.select('table.result-table')" }, { "line": 915, "expression": "rows[0].find_all('td', recursive=False)" }, { "line": 937, "expression": "table.find_all('a', href=True)" }, { "line": 947, "expression": "table.find_all('a', href=True)" }, { "line": 928, "expression": "rows[1].find_all('td', recursive=False)" }, { "line": 912, "expression": "table.find_all('tr', recursive=False)" } ], "dictionary_entries": [ { "line": 972, "key": "source_collection", "value_expression": "'mutopia'" }, { "line": 973, "key": "title", "value_expression": "title" }, { "line": 974, "key": "composer", "value_expression": "composer" }, { "line": 975, "key": "source_url", "value_expression": "piece_url or identity_url" }, { "line": 976, "key": "license_name", "value_expression": "license_name" }, { "line": 977, "key": "license_url", "value_expression": "license_url" }, { "line": 978, "key": "catalog_number", "value_expression": "catalog_number or _extract_bwv(title)" }, { "line": 979, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 980, "key": "assets", "value_expression": "_limited(assets, int(self.config.get('max_assets_per_record', 0) or 0))" }, { "line": 981, "key": "raw_data", "value_expression": "{'mutopia_listing_title': title, 'mutopia_piece_url': piece_url, 'mutopia_listing_url': source_url}" }, { "line": 954, "key": "url", "value_expression": "href" }, { "line": 955, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 956, "key": "format", "value_expression": "asset_format_from_url(href)['format']" }, { "line": 957, "key": "label", "value_expression": "label" }, { "line": 982, "key": "mutopia_listing_title", "value_expression": "title" }, { "line": 983, "key": "mutopia_piece_url", "value_expression": "piece_url" }, { "line": 984, "key": "mutopia_listing_url", "value_expression": "source_url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_assets", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 832, "line_end": 855, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 835, "receiver": "options", "key": "pdmx_enable_archive_streaming", "expression": "options.get('pdmx_enable_archive_streaming')" }, { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 850, "receiver": "options", "key": "pdmx_archive_stream_max_mb", "expression": "options.get('pdmx_archive_stream_max_mb')" } ], "html_queries": [], "dictionary_entries": [ { "line": 841, "key": "file_name", "value_expression": "Path(value).name" }, { "line": 841, "key": "format", "value_expression": "asset_format_from_url(value)['format']" }, { "line": 841, "key": "label", "value_expression": "label" } ], "subscript_assignments": [ { "line": 843, "value_line": 843, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 845, "value_line": 845, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 848, "value_line": 848, "key": "archive_url", "target_expression": "asset['archive_url']", "value_expression": "archive_url" }, { "line": 849, "value_line": 849, "key": "archive_member_path", "target_expression": "asset['archive_member_path']", "value_expression": "value.replace('./', '', 1)" }, { "line": 850, "value_line": 850, "key": "archive_stream_max_mb", "target_expression": "asset['archive_stream_max_mb']", "value_expression": "options.get('pdmx_archive_stream_max_mb')" }, { "line": 851, "value_line": 851, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'archive_stream_required'" }, { "line": 853, "value_line": 853, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'local_snapshot_missing'" } ] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_row_matches", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 814, "line_end": 830, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 821, "receiver": "options", "key": "license_filter", "expression": "options.get('license_filter')" }, { "line": 824, "receiver": "options", "key": "query", "expression": "options.get('query')" }, { "line": 826, "receiver": "options", "key": "organ_only", "expression": "options.get('organ_only')" }, { "line": 827, "receiver": "self.config", "key": "pdmx_include_track_19", "expression": "self.config.get('pdmx_include_track_19', True)" }, { "line": 815, "receiver": "row", "key": "subset:deduplicated", "expression": "row.get('subset:deduplicated', '')" }, { "line": 817, "receiver": "row", "key": "subset:no_license_conflict", "expression": "row.get('subset:no_license_conflict', '')" }, { "line": 819, "receiver": "row", "key": "subset:all_valid", "expression": "row.get('subset:all_valid', '')" }, { "line": 821, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 827, "receiver": "row", "key": "tracks", "expression": "row.get('tracks')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._raw_record", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 565, "line_end": 599, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 581, "receiver": "asset", "key": "format", "expression": "asset.get('format')" }, { "line": 581, "receiver": "asset", "key": "url", "expression": "asset.get('url')" }, { "line": 581, "receiver": "asset", "key": "local_path", "expression": "asset.get('local_path')" } ], "html_queries": [], "dictionary_entries": [ { "line": 583, "key": "score_id", "value_expression": "score_id" }, { "line": 584, "key": "source_collection", "value_expression": "source_collection" }, { "line": 585, "key": "title", "value_expression": "title" }, { "line": 586, "key": "composer", "value_expression": "composer" }, { "line": 587, "key": "source_url", "value_expression": "source_url" }, { "line": 588, "key": "url", "value_expression": "source_url" }, { "line": 589, "key": "license_name", "value_expression": "license_name" }, { "line": 590, "key": "license_url", "value_expression": "license_url" }, { "line": 591, "key": "catalog_number", "value_expression": "catalog_number" }, { "line": 592, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 593, "key": "formats", "value_expression": "','.join(formats)" }, { "line": 594, "key": "status", "value_expression": "status" }, { "line": 595, "key": "assets", "value_expression": "assets" }, { "line": 596, "key": "raw_data_json", "value_expression": "json.dumps({**raw_data, 'assets': assets}, ensure_ascii=False)" }, { "line": 597, "key": "raw_data", "value_expression": "raw_data" }, { "line": 598, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 596, "key": "assets", "value_expression": "assets" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._respectful_pause", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 1021, "line_end": 1024, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 1022, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 145, "line_end": 164, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_collection", "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "preflight_crawl" ], "literal_get_calls": [ { "line": 148, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 150, "receiver": "preflight", "key": "ok", "expression": "preflight.get('ok')" }, { "line": 151, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" }, { "line": 152, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" } ], "html_queries": [], "dictionary_entries": [ { "line": 149, "key": "collection", "value_expression": "','.join(collections)" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.preflight_crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 122, "line_end": 143, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 143, "key": "ok", "value_expression": "True" }, { "line": 125, "key": "ok", "value_expression": "False" }, { "line": 125, "key": "message", "value_expression": "'No supported digital sheet music collections were selected'" }, { "line": 140, "key": "ok", "value_expression": "False" }, { "line": 141, "key": "message", "value_expression": "'musescore_direct requires explicit URLs; an empty credential boundary is not a completed crawl'" }, { "line": 129, "key": "ok", "value_expression": "False" }, { "line": 129, "key": "message", "value_expression": "\"pdmx_mode must be either 'rows' or 'inventory'\"" }, { "line": 132, "key": "ok", "value_expression": "False" }, { "line": 134, "key": "message", "value_expression": "'PDMX row acquisition requires an existing pdmx_csv_path; use pdmx_mode=inventory only for an explicit dataset-inventory scope'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 166, "line_end": 179, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 167, "receiver": "raw_data", "key": "score_id", "expression": "raw_data.get('score_id')" }, { "line": 167, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 169, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 178, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "score_id", "value_expression": "raw_data.get('score_id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "id", "value_expression": "raw_data.get('id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 169, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "transform_run_id", "value_expression": "raw_data.get('transform_run_id')", "line": 178, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 148, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "ok", "value_expression": "preflight.get('ok')", "line": 150, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 172, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 174, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:\n pass\n return normalize_digital_sheet_music_record(" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 175, "line_end": 179, "text": " payload,\n primary_id=score_id,\n raw_content=json.dumps(payload, ensure_ascii=False, sort_keys=True),\n transform_run_id=raw_data.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 145, "line_end": 153, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 147, "line_end": 155, "text": " collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))\n records: List[Dict[str, Any]] = []\n for index, collection in enumerate(collections, start=1):" } ] } }, { "catalogue_id": "S055", "source_key": "score_orgelbuechlein", "name": "Little Organ Book / Orgelbuechlein Dataset", "selection_role": "score_collection", "parent_source_key": "digital_sheet_music", "attribution": "H. Asai; Y. Gao; S. Shimosaka; S. Tojo; Y. Uehara", "scope_description": "Bach's Orgelbüchlein", "documentary_unit": "Encoded score and harmonic-analysis version", "source_description": "The repository provides MusicXML, PDF and Roman-numeral-text material. It identifies Pierre Gouin's 2008 edition as the score source and credits the 2022 conversion, editing and annotation. Its harmonic analysis is presented as an interpretation, and a revised BWV 621 text analysis coexists with the earlier version.", "source_description_references": [ "score_orgelbuechlein" ], "interpretation": "Cite the edition, encoding and analysis separately. Corrected notation and harmonic labels are editorial contributions rather than a direct transcription of compositional intent. Pin a commit and file path for repeatable analysis.", "rights_observation": "The README states CC BY-NC 3.0 and identifies the original edition and adaptation credits.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "score_orgelbuechlein" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "This is the orgelbuechlein collection route inside DigitalSheetMusicCrawler, not a separately registered crawler. The route lists repository files and selects MusicXML extensions. The inspected code assigns Bach as composer, organ instrumentation and a licence label, and constructs a corresponding PDF URL by replacing a directory and suffix.", "transformation": "The shared score normalizer separates work metadata, assets, formats, collection context and licence statements. Those constants and the constructed PDF address are implementation values. Their correctness needs the collection documentation and an actual asset check; the existence of a generated URL does not prove a file exists. Score edition, encoding and analytical annotation remain separate.", "limitations": "Cite the edition, encoding and analysis separately. Corrected notation and harmonic labels are editorial contributions rather than a direct transcription of compositional intent. Pin a commit and file path for repeatable analysis. Availability of a collection route does not demonstrate that all pages or files were acquired." }, "implementation": { "class": "DigitalSheetMusicCrawler", "class_line": 51, "path": "aggregator/digital_sheet_music_crawler.py", "inheritance": [ "DigitalSheetMusicCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "score_id", "path": "aggregator/digital_sheet_music_crawler.py", "line": 58, "class": "DigitalSheetMusicCrawler" }, "record_label": { "value": "digital scores", "path": "aggregator/digital_sheet_music_crawler.py", "line": 59, "class": "DigitalSheetMusicCrawler" } }, "methods": [ { "symbol": "DigitalSheetMusicCrawler._assets_from_html", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 857, "line_end": 905, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 865, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 864, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 887, "key": "url", "value_expression": "href" }, { "line": 887, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 887, "key": "format", "value_expression": "info['format']" }, { "line": 887, "key": "label", "value_expression": "text or info['format']" }, { "line": 897, "key": "url", "value_expression": "href" }, { "line": 898, "key": "alternate_urls", "value_expression": "_imslp_mirror_candidates(href, imslp_file_id)" }, { "line": 899, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 900, "key": "format", "value_expression": "info['format']" }, { "line": 901, "key": "label", "value_expression": "text or info['format']" }, { "line": 902, "key": "source_file_id", "value_expression": "imslp_file_id" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_collection", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 318, "line_end": 329, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_imslp_wima", "_crawl_musescore_direct", "_crawl_mutopia", "_crawl_orgelbuechlein", "_crawl_pdmx" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_imslp_wima", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 457, "line_end": 484, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 459, "receiver": "options", "key": "imslp_seed_urls", "expression": "options.get('imslp_seed_urls')" }, { "line": 460, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 461, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 479, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [], "dictionary_entries": [ { "line": 480, "key": "html_excerpt", "value_expression": "html[:10000]" }, { "line": 480, "key": "seed_url", "value_expression": "url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_musescore_direct", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 513, "line_end": 563, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 514, "receiver": "options", "key": "musescore_direct_urls", "expression": "options.get('musescore_direct_urls')" }, { "line": 515, "receiver": "self.config", "key": "musescore_credentials_env", "expression": "self.config.get('musescore_credentials_env')" }, { "line": 539, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 543, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 532, "key": "credential_required", "value_expression": "True" }, { "line": 533, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "credential_required", "value_expression": "True" }, { "line": 559, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "http_status", "value_expression": "response.status_code" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_mutopia", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 486, "line_end": 511, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_mutopia_next_url", "_parse_mutopia_listing", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 487, "receiver": "self.config", "key": "mutopia_collection_url", "expression": "self.config.get('mutopia_collection_url')" }, { "line": 488, "receiver": "self.config", "key": "mutopia_max_pages", "expression": "self.config.get('mutopia_max_pages', 0)" }, { "line": 496, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 500, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_options", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 280, "line_end": 316, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "source", "key": "collection", "expression": "source.get('collection')" }, { "line": 286, "receiver": "source", "key": "collections", "expression": "source.get('collections')" }, { "line": 293, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 295, "receiver": "source", "key": "organ_only", "expression": "source.get('organ_only')" }, { "line": 302, "receiver": "source", "key": "pdmx_enable_archive_streaming", "expression": "source.get('pdmx_enable_archive_streaming')" }, { "line": 288, "receiver": "self.config", "key": "default_collections", "expression": "self.config.get('default_collections')" }, { "line": 297, "receiver": "source", "key": "pdmx_csv_path", "expression": "source.get('pdmx_csv_path')" }, { "line": 297, "receiver": "self.config", "key": "pdmx_csv_path", "expression": "self.config.get('pdmx_csv_path')" }, { "line": 299, "receiver": "source", "key": "pdmx_root_dir", "expression": "source.get('pdmx_root_dir')" }, { "line": 299, "receiver": "self.config", "key": "pdmx_root_dir", "expression": "self.config.get('pdmx_root_dir')" }, { "line": 300, "receiver": "source", "key": "pdmx_zenodo_record_api_url", "expression": "source.get('pdmx_zenodo_record_api_url')" }, { "line": 300, "receiver": "self.config", "key": "pdmx_zenodo_record_api_url", "expression": "self.config.get('pdmx_zenodo_record_api_url')" }, { "line": 303, "receiver": "self.config", "key": "pdmx_enable_archive_streaming", "expression": "self.config.get('pdmx_enable_archive_streaming', False)" }, { "line": 306, "receiver": "source", "key": "pdmx_archive_stream_max_mb", "expression": "source.get('pdmx_archive_stream_max_mb')" }, { "line": 306, "receiver": "self.config", "key": "pdmx_archive_stream_max_mb", "expression": "self.config.get('pdmx_archive_stream_max_mb')" }, { "line": 314, "receiver": "source", "key": "imslp_seed_urls", "expression": "source.get('imslp_seed_urls')" }, { "line": 315, "receiver": "source", "key": "musescore_direct_urls", "expression": "source.get('musescore_direct_urls')" }, { "line": 310, "receiver": "source", "key": "pdmx_mxl_archive_url", "expression": "source.get('pdmx_mxl_archive_url')" }, { "line": 310, "receiver": "self.config", "key": "pdmx_mxl_archive_url", "expression": "self.config.get('pdmx_mxl_archive_url')" }, { "line": 311, "receiver": "source", "key": "pdmx_mid_archive_url", "expression": "source.get('pdmx_mid_archive_url')" }, { "line": 311, "receiver": "self.config", "key": "pdmx_mid_archive_url", "expression": "self.config.get('pdmx_mid_archive_url')" }, { "line": 312, "receiver": "source", "key": "pdmx_pdf_archive_url", "expression": "source.get('pdmx_pdf_archive_url')" }, { "line": 312, "receiver": "self.config", "key": "pdmx_pdf_archive_url", "expression": "self.config.get('pdmx_pdf_archive_url')" }, { "line": 314, "receiver": "self.config", "key": "imslp_seed_urls", "expression": "self.config.get('imslp_seed_urls')" }, { "line": 315, "receiver": "self.config", "key": "musescore_direct_urls", "expression": "self.config.get('musescore_direct_urls')" }, { "line": 294, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 296, "receiver": "source", "key": "license_filter", "expression": "source.get('license_filter')" }, { "line": 298, "receiver": "source", "key": "pdmx_mode", "expression": "source.get('pdmx_mode')" }, { "line": 298, "receiver": "self.config", "key": "pdmx_mode", "expression": "self.config.get('pdmx_mode')" } ], "html_queries": [], "dictionary_entries": [ { "line": 292, "key": "collections", "value_expression": "[collection for collection in collections if collection in SOURCE_COLLECTIONS]" }, { "line": 293, "key": "limit", "value_expression": "_safe_int(source.get('limit'), default=0)" }, { "line": 294, "key": "query", "value_expression": "str(source.get('query') or '').strip().lower()" }, { "line": 295, "key": "organ_only", "value_expression": "_read_bool(source.get('organ_only'), True)" }, { "line": 296, "key": "license_filter", "value_expression": "str(source.get('license_filter') or '').strip().lower()" }, { "line": 297, "key": "pdmx_csv_path", "value_expression": "str(source.get('pdmx_csv_path') or self.config.get('pdmx_csv_path') or '')" }, { "line": 298, "key": "pdmx_mode", "value_expression": "str(source.get('pdmx_mode') or self.config.get('pdmx_mode') or 'rows').strip().lower()" }, { "line": 299, "key": "pdmx_root_dir", "value_expression": "str(source.get('pdmx_root_dir') or self.config.get('pdmx_root_dir') or '')" }, { "line": 300, "key": "pdmx_zenodo_record_api_url", "value_expression": "str(source.get('pdmx_zenodo_record_api_url') or self.config.get('pdmx_zenodo_record_api_url') or '')" }, { "line": 301, "key": "pdmx_enable_archive_streaming", "value_expression": "_read_bool(source.get('pdmx_enable_archive_streaming'), bool(self.config.get('pdmx_enable_archive_streaming', False)))" }, { "line": 305, "key": "pdmx_archive_stream_max_mb", "value_expression": "_safe_int(source.get('pdmx_archive_stream_max_mb') or self.config.get('pdmx_archive_stream_max_mb'), default=256)" }, { "line": 309, "key": "pdmx_archive_urls", "value_expression": "{'mxl': str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or ''), 'mid': str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or ''), 'pdf': str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')}" }, { "line": 314, "key": "imslp_seed_urls", "value_expression": "_split_lines(source.get('imslp_seed_urls')) or list(self.config.get('imslp_seed_urls') or [])" }, { "line": 315, "key": "musescore_direct_urls", "value_expression": "_split_lines(source.get('musescore_direct_urls')) or list(self.config.get('musescore_direct_urls') or [])" }, { "line": 310, "key": "mxl", "value_expression": "str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or '')" }, { "line": 311, "key": "mid", "value_expression": "str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or '')" }, { "line": 312, "key": "pdf", "value_expression": "str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')" } ], "subscript_assignments": [ { "line": 285, "value_line": 285, "key": "collection", "target_expression": "source['collection']", "value_expression": "str(target)" } ] }, { "symbol": "DigitalSheetMusicCrawler._crawl_orgelbuechlein", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 416, "line_end": 455, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 417, "receiver": "self.config", "key": "orgelbuechlein_api_url", "expression": "self.config.get('orgelbuechlein_api_url')" }, { "line": 429, "receiver": "item", "key": "download_url", "expression": "item.get('download_url')" }, { "line": 429, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 418, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 423, "receiver": "item", "key": "name", "expression": "item.get('name')" }, { "line": 436, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 453, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "github_item", "value_expression": "item" }, { "line": 442, "key": "url", "value_expression": "raw_url" }, { "line": 442, "key": "file_name", "value_expression": "name" }, { "line": 442, "key": "format", "value_expression": "asset_format_from_url(name)['format']" }, { "line": 442, "key": "label", "value_expression": "'MusicXML'" }, { "line": 444, "key": "url", "value_expression": "raw_url.replace('/little-organ-book-mxl/', '/little-organ-book-pdf/').rsplit('.', 1)[0] + '.pdf'" }, { "line": 445, "key": "file_name", "value_expression": "f'{stem}.pdf'" }, { "line": 446, "key": "format", "value_expression": "'pdf'" }, { "line": 447, "key": "label", "value_expression": "'PDF'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 331, "line_end": 364, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_pdmx_zenodo_inventory", "_pdmx_assets", "_pdmx_row_matches", "_raw_record", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 332, "receiver": "options", "key": "pdmx_mode", "expression": "options.get('pdmx_mode')" }, { "line": 338, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 334, "receiver": "options", "key": "pdmx_csv_path", "expression": "options.get('pdmx_csv_path')" }, { "line": 337, "receiver": "options", "key": "pdmx_root_dir", "expression": "options.get('pdmx_root_dir')" }, { "line": 338, "receiver": "self.config", "key": "pdmx_max_rows", "expression": "self.config.get('pdmx_max_rows')" }, { "line": 348, "receiver": "row", "key": "path", "expression": "row.get('path')" }, { "line": 348, "receiver": "row", "key": "mxl", "expression": "row.get('mxl')" }, { "line": 348, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "song_name", "expression": "row.get('song_name')" }, { "line": 354, "receiver": "row", "key": "composer_name", "expression": "row.get('composer_name')" }, { "line": 354, "receiver": "row", "key": "artist_name", "expression": "row.get('artist_name')" }, { "line": 356, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 357, "receiver": "row", "key": "license_url", "expression": "row.get('license_url')" }, { "line": 359, "receiver": "row", "key": "instrumentation", "expression": "row.get('instrumentation')" }, { "line": 359, "receiver": "row", "key": "instruments", "expression": "row.get('instruments')" }, { "line": 359, "receiver": "row", "key": "instrument", "expression": "row.get('instrument')" } ], "html_queries": [], "dictionary_entries": [ { "line": 361, "key": "pdmx", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx_zenodo_inventory", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 366, "line_end": 414, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 373, "receiver": "payload", "key": "files", "expression": "payload.get('files')" }, { "line": 390, "receiver": "payload.get('metadata', {})", "key": "title", "expression": "payload.get('metadata', {}).get('title')" }, { "line": 367, "receiver": "options", "key": "pdmx_zenodo_record_api_url", "expression": "options.get('pdmx_zenodo_record_api_url')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "content", "expression": "item.get('links', {}).get('content')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "self", "expression": "item.get('links', {}).get('self')" }, { "line": 370, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 376, "receiver": "item", "key": "key", "expression": "item.get('key')" }, { "line": 386, "receiver": "item", "key": "size", "expression": "item.get('size')" }, { "line": 390, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 397, "receiver": "payload.get('links', {})", "key": "html", "expression": "payload.get('links', {}).get('html')" }, { "line": 398, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "id", "expression": "payload.get('metadata', {}).get('license', {}).get('id')" }, { "line": 399, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "url", "expression": "payload.get('metadata', {}).get('license', {}).get('url')" }, { "line": 406, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 407, "receiver": "payload", "key": "doi", "expression": "payload.get('doi')" }, { "line": 393, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 397, "receiver": "payload", "key": "links", "expression": "payload.get('links', {})" }, { "line": 398, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 399, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 398, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 399, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 382, "key": "url", "value_expression": "content_url" }, { "line": 383, "key": "file_name", "value_expression": "key" }, { "line": 384, "key": "format", "value_expression": "asset_format_from_url(key)['format']" }, { "line": 385, "key": "label", "value_expression": "'Zenodo dataset file'" }, { "line": 386, "key": "content_size_bytes", "value_expression": "item.get('size')" }, { "line": 387, "key": "download_status", "value_expression": "'dataset_snapshot_required'" }, { "line": 405, "key": "zenodo_record", "value_expression": "{'id': payload.get('id'), 'doi': payload.get('doi'), 'api_url': api_url, 'files': files}" }, { "line": 411, "key": "access_note", "value_expression": "'PDMX requires an explicit local snapshot path for score-level crawling to avoid automatic multi-GB dataset downloads.'" }, { "line": 406, "key": "id", "value_expression": "payload.get('id')" }, { "line": 407, "key": "doi", "value_expression": "payload.get('doi')" }, { "line": 408, "key": "api_url", "value_expression": "api_url" }, { "line": 409, "key": "files", "value_expression": "files" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._extract_tar_gz_member_from_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 780, "line_end": 812, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 785, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._mutopia_next_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 990, "line_end": 995, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 994, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 992, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._parse_mutopia_listing", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 907, "line_end": 988, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 939, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 951, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 912, "receiver": "row", "key": "class", "expression": "row.get('class')" }, { "line": 980, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [ { "line": 911, "expression": "soup.select('table.result-table')" }, { "line": 915, "expression": "rows[0].find_all('td', recursive=False)" }, { "line": 937, "expression": "table.find_all('a', href=True)" }, { "line": 947, "expression": "table.find_all('a', href=True)" }, { "line": 928, "expression": "rows[1].find_all('td', recursive=False)" }, { "line": 912, "expression": "table.find_all('tr', recursive=False)" } ], "dictionary_entries": [ { "line": 972, "key": "source_collection", "value_expression": "'mutopia'" }, { "line": 973, "key": "title", "value_expression": "title" }, { "line": 974, "key": "composer", "value_expression": "composer" }, { "line": 975, "key": "source_url", "value_expression": "piece_url or identity_url" }, { "line": 976, "key": "license_name", "value_expression": "license_name" }, { "line": 977, "key": "license_url", "value_expression": "license_url" }, { "line": 978, "key": "catalog_number", "value_expression": "catalog_number or _extract_bwv(title)" }, { "line": 979, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 980, "key": "assets", "value_expression": "_limited(assets, int(self.config.get('max_assets_per_record', 0) or 0))" }, { "line": 981, "key": "raw_data", "value_expression": "{'mutopia_listing_title': title, 'mutopia_piece_url': piece_url, 'mutopia_listing_url': source_url}" }, { "line": 954, "key": "url", "value_expression": "href" }, { "line": 955, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 956, "key": "format", "value_expression": "asset_format_from_url(href)['format']" }, { "line": 957, "key": "label", "value_expression": "label" }, { "line": 982, "key": "mutopia_listing_title", "value_expression": "title" }, { "line": 983, "key": "mutopia_piece_url", "value_expression": "piece_url" }, { "line": 984, "key": "mutopia_listing_url", "value_expression": "source_url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_assets", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 832, "line_end": 855, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 835, "receiver": "options", "key": "pdmx_enable_archive_streaming", "expression": "options.get('pdmx_enable_archive_streaming')" }, { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 850, "receiver": "options", "key": "pdmx_archive_stream_max_mb", "expression": "options.get('pdmx_archive_stream_max_mb')" } ], "html_queries": [], "dictionary_entries": [ { "line": 841, "key": "file_name", "value_expression": "Path(value).name" }, { "line": 841, "key": "format", "value_expression": "asset_format_from_url(value)['format']" }, { "line": 841, "key": "label", "value_expression": "label" } ], "subscript_assignments": [ { "line": 843, "value_line": 843, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 845, "value_line": 845, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 848, "value_line": 848, "key": "archive_url", "target_expression": "asset['archive_url']", "value_expression": "archive_url" }, { "line": 849, "value_line": 849, "key": "archive_member_path", "target_expression": "asset['archive_member_path']", "value_expression": "value.replace('./', '', 1)" }, { "line": 850, "value_line": 850, "key": "archive_stream_max_mb", "target_expression": "asset['archive_stream_max_mb']", "value_expression": "options.get('pdmx_archive_stream_max_mb')" }, { "line": 851, "value_line": 851, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'archive_stream_required'" }, { "line": 853, "value_line": 853, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'local_snapshot_missing'" } ] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_row_matches", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 814, "line_end": 830, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 821, "receiver": "options", "key": "license_filter", "expression": "options.get('license_filter')" }, { "line": 824, "receiver": "options", "key": "query", "expression": "options.get('query')" }, { "line": 826, "receiver": "options", "key": "organ_only", "expression": "options.get('organ_only')" }, { "line": 827, "receiver": "self.config", "key": "pdmx_include_track_19", "expression": "self.config.get('pdmx_include_track_19', True)" }, { "line": 815, "receiver": "row", "key": "subset:deduplicated", "expression": "row.get('subset:deduplicated', '')" }, { "line": 817, "receiver": "row", "key": "subset:no_license_conflict", "expression": "row.get('subset:no_license_conflict', '')" }, { "line": 819, "receiver": "row", "key": "subset:all_valid", "expression": "row.get('subset:all_valid', '')" }, { "line": 821, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 827, "receiver": "row", "key": "tracks", "expression": "row.get('tracks')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._raw_record", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 565, "line_end": 599, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 581, "receiver": "asset", "key": "format", "expression": "asset.get('format')" }, { "line": 581, "receiver": "asset", "key": "url", "expression": "asset.get('url')" }, { "line": 581, "receiver": "asset", "key": "local_path", "expression": "asset.get('local_path')" } ], "html_queries": [], "dictionary_entries": [ { "line": 583, "key": "score_id", "value_expression": "score_id" }, { "line": 584, "key": "source_collection", "value_expression": "source_collection" }, { "line": 585, "key": "title", "value_expression": "title" }, { "line": 586, "key": "composer", "value_expression": "composer" }, { "line": 587, "key": "source_url", "value_expression": "source_url" }, { "line": 588, "key": "url", "value_expression": "source_url" }, { "line": 589, "key": "license_name", "value_expression": "license_name" }, { "line": 590, "key": "license_url", "value_expression": "license_url" }, { "line": 591, "key": "catalog_number", "value_expression": "catalog_number" }, { "line": 592, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 593, "key": "formats", "value_expression": "','.join(formats)" }, { "line": 594, "key": "status", "value_expression": "status" }, { "line": 595, "key": "assets", "value_expression": "assets" }, { "line": 596, "key": "raw_data_json", "value_expression": "json.dumps({**raw_data, 'assets': assets}, ensure_ascii=False)" }, { "line": 597, "key": "raw_data", "value_expression": "raw_data" }, { "line": 598, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 596, "key": "assets", "value_expression": "assets" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._respectful_pause", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 1021, "line_end": 1024, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 1022, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 145, "line_end": 164, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_collection", "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "preflight_crawl" ], "literal_get_calls": [ { "line": 148, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 150, "receiver": "preflight", "key": "ok", "expression": "preflight.get('ok')" }, { "line": 151, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" }, { "line": 152, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" } ], "html_queries": [], "dictionary_entries": [ { "line": 149, "key": "collection", "value_expression": "','.join(collections)" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.preflight_crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 122, "line_end": 143, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 143, "key": "ok", "value_expression": "True" }, { "line": 125, "key": "ok", "value_expression": "False" }, { "line": 125, "key": "message", "value_expression": "'No supported digital sheet music collections were selected'" }, { "line": 140, "key": "ok", "value_expression": "False" }, { "line": 141, "key": "message", "value_expression": "'musescore_direct requires explicit URLs; an empty credential boundary is not a completed crawl'" }, { "line": 129, "key": "ok", "value_expression": "False" }, { "line": 129, "key": "message", "value_expression": "\"pdmx_mode must be either 'rows' or 'inventory'\"" }, { "line": 132, "key": "ok", "value_expression": "False" }, { "line": 134, "key": "message", "value_expression": "'PDMX row acquisition requires an existing pdmx_csv_path; use pdmx_mode=inventory only for an explicit dataset-inventory scope'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 166, "line_end": 179, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 167, "receiver": "raw_data", "key": "score_id", "expression": "raw_data.get('score_id')" }, { "line": 167, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 169, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 178, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "score_id", "value_expression": "raw_data.get('score_id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "id", "value_expression": "raw_data.get('id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 169, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "transform_run_id", "value_expression": "raw_data.get('transform_run_id')", "line": 178, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 148, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "ok", "value_expression": "preflight.get('ok')", "line": 150, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 172, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 174, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:\n pass\n return normalize_digital_sheet_music_record(" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 175, "line_end": 179, "text": " payload,\n primary_id=score_id,\n raw_content=json.dumps(payload, ensure_ascii=False, sort_keys=True),\n transform_run_id=raw_data.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 145, "line_end": 153, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 147, "line_end": 155, "text": " collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))\n records: List[Dict[str, Any]] = []\n for index, collection in enumerate(collections, start=1):" } ] } }, { "catalogue_id": "S056", "source_key": "score_imslp_wima", "name": "IMSLP / Werner Icking Music Collection", "selection_role": "score_collection", "parent_source_key": "digital_sheet_music", "attribution": "IMSLP / Petrucci Music Library; Werner Icking Music Archive contributors", "scope_description": "Broad score collections; organ-relevant selections", "documentary_unit": "Edition or file attached to a work page", "source_description": "Aggregator groups IMSLP and WIMA under one collection key and uses selected work pages. The MusicXML resource directory notes that the Werner Icking archive merged with IMSLP while its original site remains available.", "source_description_references": [ "score_imslp_wima", "digital_sheet_music", "repo_score_collections" ], "interpretation": "A work page can host several editions, arrangements and file formats. Cite the specific file, editor or contributor and edition date; a work title alone is insufficient. WIMA provenance should survive the hosting relationship.", "rights_observation": "Rights must be read for the specific edition, file and applicable jurisdiction. This review does not assign a collection-wide licence.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "score_imslp_wima", "digital_sheet_music", "repo_score_collections" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "This is the imslp_wima collection route inside DigitalSheetMusicCrawler, not a separately registered crawler. The collection route handles selected work-page material and uses the shared HTML asset finder. It is a route to selected pages, not a complete mirror of IMSLP or the historical WIMA site.", "transformation": "The shared score normalizer separates work metadata, assets, formats, collection context and licence statements. A work page may contain multiple editions, arrangements and file types. The collection key is not an umbrella rights statement or a stable edition identifier.", "limitations": "A work page can host several editions, arrangements and file formats. Cite the specific file, editor or contributor and edition date; a work title alone is insufficient. WIMA provenance should survive the hosting relationship. Availability of a collection route does not demonstrate that all pages or files were acquired." }, "implementation": { "class": "DigitalSheetMusicCrawler", "class_line": 51, "path": "aggregator/digital_sheet_music_crawler.py", "inheritance": [ "DigitalSheetMusicCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "score_id", "path": "aggregator/digital_sheet_music_crawler.py", "line": 58, "class": "DigitalSheetMusicCrawler" }, "record_label": { "value": "digital scores", "path": "aggregator/digital_sheet_music_crawler.py", "line": 59, "class": "DigitalSheetMusicCrawler" } }, "methods": [ { "symbol": "DigitalSheetMusicCrawler._assets_from_html", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 857, "line_end": 905, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 865, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 864, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 887, "key": "url", "value_expression": "href" }, { "line": 887, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 887, "key": "format", "value_expression": "info['format']" }, { "line": 887, "key": "label", "value_expression": "text or info['format']" }, { "line": 897, "key": "url", "value_expression": "href" }, { "line": 898, "key": "alternate_urls", "value_expression": "_imslp_mirror_candidates(href, imslp_file_id)" }, { "line": 899, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 900, "key": "format", "value_expression": "info['format']" }, { "line": 901, "key": "label", "value_expression": "text or info['format']" }, { "line": 902, "key": "source_file_id", "value_expression": "imslp_file_id" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_collection", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 318, "line_end": 329, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_imslp_wima", "_crawl_musescore_direct", "_crawl_mutopia", "_crawl_orgelbuechlein", "_crawl_pdmx" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_imslp_wima", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 457, "line_end": 484, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 459, "receiver": "options", "key": "imslp_seed_urls", "expression": "options.get('imslp_seed_urls')" }, { "line": 460, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 461, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 479, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [], "dictionary_entries": [ { "line": 480, "key": "html_excerpt", "value_expression": "html[:10000]" }, { "line": 480, "key": "seed_url", "value_expression": "url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_musescore_direct", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 513, "line_end": 563, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 514, "receiver": "options", "key": "musescore_direct_urls", "expression": "options.get('musescore_direct_urls')" }, { "line": 515, "receiver": "self.config", "key": "musescore_credentials_env", "expression": "self.config.get('musescore_credentials_env')" }, { "line": 539, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 543, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 532, "key": "credential_required", "value_expression": "True" }, { "line": 533, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "credential_required", "value_expression": "True" }, { "line": 559, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "http_status", "value_expression": "response.status_code" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_mutopia", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 486, "line_end": 511, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_mutopia_next_url", "_parse_mutopia_listing", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 487, "receiver": "self.config", "key": "mutopia_collection_url", "expression": "self.config.get('mutopia_collection_url')" }, { "line": 488, "receiver": "self.config", "key": "mutopia_max_pages", "expression": "self.config.get('mutopia_max_pages', 0)" }, { "line": 496, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 500, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_options", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 280, "line_end": 316, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "source", "key": "collection", "expression": "source.get('collection')" }, { "line": 286, "receiver": "source", "key": "collections", "expression": "source.get('collections')" }, { "line": 293, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 295, "receiver": "source", "key": "organ_only", "expression": "source.get('organ_only')" }, { "line": 302, "receiver": "source", "key": "pdmx_enable_archive_streaming", "expression": "source.get('pdmx_enable_archive_streaming')" }, { "line": 288, "receiver": "self.config", "key": "default_collections", "expression": "self.config.get('default_collections')" }, { "line": 297, "receiver": "source", "key": "pdmx_csv_path", "expression": "source.get('pdmx_csv_path')" }, { "line": 297, "receiver": "self.config", "key": "pdmx_csv_path", "expression": "self.config.get('pdmx_csv_path')" }, { "line": 299, "receiver": "source", "key": "pdmx_root_dir", "expression": "source.get('pdmx_root_dir')" }, { "line": 299, "receiver": "self.config", "key": "pdmx_root_dir", "expression": "self.config.get('pdmx_root_dir')" }, { "line": 300, "receiver": "source", "key": "pdmx_zenodo_record_api_url", "expression": "source.get('pdmx_zenodo_record_api_url')" }, { "line": 300, "receiver": "self.config", "key": "pdmx_zenodo_record_api_url", "expression": "self.config.get('pdmx_zenodo_record_api_url')" }, { "line": 303, "receiver": "self.config", "key": "pdmx_enable_archive_streaming", "expression": "self.config.get('pdmx_enable_archive_streaming', False)" }, { "line": 306, "receiver": "source", "key": "pdmx_archive_stream_max_mb", "expression": "source.get('pdmx_archive_stream_max_mb')" }, { "line": 306, "receiver": "self.config", "key": "pdmx_archive_stream_max_mb", "expression": "self.config.get('pdmx_archive_stream_max_mb')" }, { "line": 314, "receiver": "source", "key": "imslp_seed_urls", "expression": "source.get('imslp_seed_urls')" }, { "line": 315, "receiver": "source", "key": "musescore_direct_urls", "expression": "source.get('musescore_direct_urls')" }, { "line": 310, "receiver": "source", "key": "pdmx_mxl_archive_url", "expression": "source.get('pdmx_mxl_archive_url')" }, { "line": 310, "receiver": "self.config", "key": "pdmx_mxl_archive_url", "expression": "self.config.get('pdmx_mxl_archive_url')" }, { "line": 311, "receiver": "source", "key": "pdmx_mid_archive_url", "expression": "source.get('pdmx_mid_archive_url')" }, { "line": 311, "receiver": "self.config", "key": "pdmx_mid_archive_url", "expression": "self.config.get('pdmx_mid_archive_url')" }, { "line": 312, "receiver": "source", "key": "pdmx_pdf_archive_url", "expression": "source.get('pdmx_pdf_archive_url')" }, { "line": 312, "receiver": "self.config", "key": "pdmx_pdf_archive_url", "expression": "self.config.get('pdmx_pdf_archive_url')" }, { "line": 314, "receiver": "self.config", "key": "imslp_seed_urls", "expression": "self.config.get('imslp_seed_urls')" }, { "line": 315, "receiver": "self.config", "key": "musescore_direct_urls", "expression": "self.config.get('musescore_direct_urls')" }, { "line": 294, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 296, "receiver": "source", "key": "license_filter", "expression": "source.get('license_filter')" }, { "line": 298, "receiver": "source", "key": "pdmx_mode", "expression": "source.get('pdmx_mode')" }, { "line": 298, "receiver": "self.config", "key": "pdmx_mode", "expression": "self.config.get('pdmx_mode')" } ], "html_queries": [], "dictionary_entries": [ { "line": 292, "key": "collections", "value_expression": "[collection for collection in collections if collection in SOURCE_COLLECTIONS]" }, { "line": 293, "key": "limit", "value_expression": "_safe_int(source.get('limit'), default=0)" }, { "line": 294, "key": "query", "value_expression": "str(source.get('query') or '').strip().lower()" }, { "line": 295, "key": "organ_only", "value_expression": "_read_bool(source.get('organ_only'), True)" }, { "line": 296, "key": "license_filter", "value_expression": "str(source.get('license_filter') or '').strip().lower()" }, { "line": 297, "key": "pdmx_csv_path", "value_expression": "str(source.get('pdmx_csv_path') or self.config.get('pdmx_csv_path') or '')" }, { "line": 298, "key": "pdmx_mode", "value_expression": "str(source.get('pdmx_mode') or self.config.get('pdmx_mode') or 'rows').strip().lower()" }, { "line": 299, "key": "pdmx_root_dir", "value_expression": "str(source.get('pdmx_root_dir') or self.config.get('pdmx_root_dir') or '')" }, { "line": 300, "key": "pdmx_zenodo_record_api_url", "value_expression": "str(source.get('pdmx_zenodo_record_api_url') or self.config.get('pdmx_zenodo_record_api_url') or '')" }, { "line": 301, "key": "pdmx_enable_archive_streaming", "value_expression": "_read_bool(source.get('pdmx_enable_archive_streaming'), bool(self.config.get('pdmx_enable_archive_streaming', False)))" }, { "line": 305, "key": "pdmx_archive_stream_max_mb", "value_expression": "_safe_int(source.get('pdmx_archive_stream_max_mb') or self.config.get('pdmx_archive_stream_max_mb'), default=256)" }, { "line": 309, "key": "pdmx_archive_urls", "value_expression": "{'mxl': str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or ''), 'mid': str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or ''), 'pdf': str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')}" }, { "line": 314, "key": "imslp_seed_urls", "value_expression": "_split_lines(source.get('imslp_seed_urls')) or list(self.config.get('imslp_seed_urls') or [])" }, { "line": 315, "key": "musescore_direct_urls", "value_expression": "_split_lines(source.get('musescore_direct_urls')) or list(self.config.get('musescore_direct_urls') or [])" }, { "line": 310, "key": "mxl", "value_expression": "str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or '')" }, { "line": 311, "key": "mid", "value_expression": "str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or '')" }, { "line": 312, "key": "pdf", "value_expression": "str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')" } ], "subscript_assignments": [ { "line": 285, "value_line": 285, "key": "collection", "target_expression": "source['collection']", "value_expression": "str(target)" } ] }, { "symbol": "DigitalSheetMusicCrawler._crawl_orgelbuechlein", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 416, "line_end": 455, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 417, "receiver": "self.config", "key": "orgelbuechlein_api_url", "expression": "self.config.get('orgelbuechlein_api_url')" }, { "line": 429, "receiver": "item", "key": "download_url", "expression": "item.get('download_url')" }, { "line": 429, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 418, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 423, "receiver": "item", "key": "name", "expression": "item.get('name')" }, { "line": 436, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 453, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "github_item", "value_expression": "item" }, { "line": 442, "key": "url", "value_expression": "raw_url" }, { "line": 442, "key": "file_name", "value_expression": "name" }, { "line": 442, "key": "format", "value_expression": "asset_format_from_url(name)['format']" }, { "line": 442, "key": "label", "value_expression": "'MusicXML'" }, { "line": 444, "key": "url", "value_expression": "raw_url.replace('/little-organ-book-mxl/', '/little-organ-book-pdf/').rsplit('.', 1)[0] + '.pdf'" }, { "line": 445, "key": "file_name", "value_expression": "f'{stem}.pdf'" }, { "line": 446, "key": "format", "value_expression": "'pdf'" }, { "line": 447, "key": "label", "value_expression": "'PDF'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 331, "line_end": 364, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_pdmx_zenodo_inventory", "_pdmx_assets", "_pdmx_row_matches", "_raw_record", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 332, "receiver": "options", "key": "pdmx_mode", "expression": "options.get('pdmx_mode')" }, { "line": 338, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 334, "receiver": "options", "key": "pdmx_csv_path", "expression": "options.get('pdmx_csv_path')" }, { "line": 337, "receiver": "options", "key": "pdmx_root_dir", "expression": "options.get('pdmx_root_dir')" }, { "line": 338, "receiver": "self.config", "key": "pdmx_max_rows", "expression": "self.config.get('pdmx_max_rows')" }, { "line": 348, "receiver": "row", "key": "path", "expression": "row.get('path')" }, { "line": 348, "receiver": "row", "key": "mxl", "expression": "row.get('mxl')" }, { "line": 348, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "song_name", "expression": "row.get('song_name')" }, { "line": 354, "receiver": "row", "key": "composer_name", "expression": "row.get('composer_name')" }, { "line": 354, "receiver": "row", "key": "artist_name", "expression": "row.get('artist_name')" }, { "line": 356, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 357, "receiver": "row", "key": "license_url", "expression": "row.get('license_url')" }, { "line": 359, "receiver": "row", "key": "instrumentation", "expression": "row.get('instrumentation')" }, { "line": 359, "receiver": "row", "key": "instruments", "expression": "row.get('instruments')" }, { "line": 359, "receiver": "row", "key": "instrument", "expression": "row.get('instrument')" } ], "html_queries": [], "dictionary_entries": [ { "line": 361, "key": "pdmx", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx_zenodo_inventory", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 366, "line_end": 414, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 373, "receiver": "payload", "key": "files", "expression": "payload.get('files')" }, { "line": 390, "receiver": "payload.get('metadata', {})", "key": "title", "expression": "payload.get('metadata', {}).get('title')" }, { "line": 367, "receiver": "options", "key": "pdmx_zenodo_record_api_url", "expression": "options.get('pdmx_zenodo_record_api_url')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "content", "expression": "item.get('links', {}).get('content')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "self", "expression": "item.get('links', {}).get('self')" }, { "line": 370, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 376, "receiver": "item", "key": "key", "expression": "item.get('key')" }, { "line": 386, "receiver": "item", "key": "size", "expression": "item.get('size')" }, { "line": 390, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 397, "receiver": "payload.get('links', {})", "key": "html", "expression": "payload.get('links', {}).get('html')" }, { "line": 398, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "id", "expression": "payload.get('metadata', {}).get('license', {}).get('id')" }, { "line": 399, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "url", "expression": "payload.get('metadata', {}).get('license', {}).get('url')" }, { "line": 406, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 407, "receiver": "payload", "key": "doi", "expression": "payload.get('doi')" }, { "line": 393, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 397, "receiver": "payload", "key": "links", "expression": "payload.get('links', {})" }, { "line": 398, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 399, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 398, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 399, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 382, "key": "url", "value_expression": "content_url" }, { "line": 383, "key": "file_name", "value_expression": "key" }, { "line": 384, "key": "format", "value_expression": "asset_format_from_url(key)['format']" }, { "line": 385, "key": "label", "value_expression": "'Zenodo dataset file'" }, { "line": 386, "key": "content_size_bytes", "value_expression": "item.get('size')" }, { "line": 387, "key": "download_status", "value_expression": "'dataset_snapshot_required'" }, { "line": 405, "key": "zenodo_record", "value_expression": "{'id': payload.get('id'), 'doi': payload.get('doi'), 'api_url': api_url, 'files': files}" }, { "line": 411, "key": "access_note", "value_expression": "'PDMX requires an explicit local snapshot path for score-level crawling to avoid automatic multi-GB dataset downloads.'" }, { "line": 406, "key": "id", "value_expression": "payload.get('id')" }, { "line": 407, "key": "doi", "value_expression": "payload.get('doi')" }, { "line": 408, "key": "api_url", "value_expression": "api_url" }, { "line": 409, "key": "files", "value_expression": "files" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._extract_tar_gz_member_from_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 780, "line_end": 812, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 785, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._mutopia_next_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 990, "line_end": 995, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 994, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 992, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._parse_mutopia_listing", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 907, "line_end": 988, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 939, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 951, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 912, "receiver": "row", "key": "class", "expression": "row.get('class')" }, { "line": 980, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [ { "line": 911, "expression": "soup.select('table.result-table')" }, { "line": 915, "expression": "rows[0].find_all('td', recursive=False)" }, { "line": 937, "expression": "table.find_all('a', href=True)" }, { "line": 947, "expression": "table.find_all('a', href=True)" }, { "line": 928, "expression": "rows[1].find_all('td', recursive=False)" }, { "line": 912, "expression": "table.find_all('tr', recursive=False)" } ], "dictionary_entries": [ { "line": 972, "key": "source_collection", "value_expression": "'mutopia'" }, { "line": 973, "key": "title", "value_expression": "title" }, { "line": 974, "key": "composer", "value_expression": "composer" }, { "line": 975, "key": "source_url", "value_expression": "piece_url or identity_url" }, { "line": 976, "key": "license_name", "value_expression": "license_name" }, { "line": 977, "key": "license_url", "value_expression": "license_url" }, { "line": 978, "key": "catalog_number", "value_expression": "catalog_number or _extract_bwv(title)" }, { "line": 979, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 980, "key": "assets", "value_expression": "_limited(assets, int(self.config.get('max_assets_per_record', 0) or 0))" }, { "line": 981, "key": "raw_data", "value_expression": "{'mutopia_listing_title': title, 'mutopia_piece_url': piece_url, 'mutopia_listing_url': source_url}" }, { "line": 954, "key": "url", "value_expression": "href" }, { "line": 955, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 956, "key": "format", "value_expression": "asset_format_from_url(href)['format']" }, { "line": 957, "key": "label", "value_expression": "label" }, { "line": 982, "key": "mutopia_listing_title", "value_expression": "title" }, { "line": 983, "key": "mutopia_piece_url", "value_expression": "piece_url" }, { "line": 984, "key": "mutopia_listing_url", "value_expression": "source_url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_assets", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 832, "line_end": 855, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 835, "receiver": "options", "key": "pdmx_enable_archive_streaming", "expression": "options.get('pdmx_enable_archive_streaming')" }, { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 850, "receiver": "options", "key": "pdmx_archive_stream_max_mb", "expression": "options.get('pdmx_archive_stream_max_mb')" } ], "html_queries": [], "dictionary_entries": [ { "line": 841, "key": "file_name", "value_expression": "Path(value).name" }, { "line": 841, "key": "format", "value_expression": "asset_format_from_url(value)['format']" }, { "line": 841, "key": "label", "value_expression": "label" } ], "subscript_assignments": [ { "line": 843, "value_line": 843, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 845, "value_line": 845, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 848, "value_line": 848, "key": "archive_url", "target_expression": "asset['archive_url']", "value_expression": "archive_url" }, { "line": 849, "value_line": 849, "key": "archive_member_path", "target_expression": "asset['archive_member_path']", "value_expression": "value.replace('./', '', 1)" }, { "line": 850, "value_line": 850, "key": "archive_stream_max_mb", "target_expression": "asset['archive_stream_max_mb']", "value_expression": "options.get('pdmx_archive_stream_max_mb')" }, { "line": 851, "value_line": 851, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'archive_stream_required'" }, { "line": 853, "value_line": 853, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'local_snapshot_missing'" } ] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_row_matches", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 814, "line_end": 830, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 821, "receiver": "options", "key": "license_filter", "expression": "options.get('license_filter')" }, { "line": 824, "receiver": "options", "key": "query", "expression": "options.get('query')" }, { "line": 826, "receiver": "options", "key": "organ_only", "expression": "options.get('organ_only')" }, { "line": 827, "receiver": "self.config", "key": "pdmx_include_track_19", "expression": "self.config.get('pdmx_include_track_19', True)" }, { "line": 815, "receiver": "row", "key": "subset:deduplicated", "expression": "row.get('subset:deduplicated', '')" }, { "line": 817, "receiver": "row", "key": "subset:no_license_conflict", "expression": "row.get('subset:no_license_conflict', '')" }, { "line": 819, "receiver": "row", "key": "subset:all_valid", "expression": "row.get('subset:all_valid', '')" }, { "line": 821, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 827, "receiver": "row", "key": "tracks", "expression": "row.get('tracks')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._raw_record", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 565, "line_end": 599, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 581, "receiver": "asset", "key": "format", "expression": "asset.get('format')" }, { "line": 581, "receiver": "asset", "key": "url", "expression": "asset.get('url')" }, { "line": 581, "receiver": "asset", "key": "local_path", "expression": "asset.get('local_path')" } ], "html_queries": [], "dictionary_entries": [ { "line": 583, "key": "score_id", "value_expression": "score_id" }, { "line": 584, "key": "source_collection", "value_expression": "source_collection" }, { "line": 585, "key": "title", "value_expression": "title" }, { "line": 586, "key": "composer", "value_expression": "composer" }, { "line": 587, "key": "source_url", "value_expression": "source_url" }, { "line": 588, "key": "url", "value_expression": "source_url" }, { "line": 589, "key": "license_name", "value_expression": "license_name" }, { "line": 590, "key": "license_url", "value_expression": "license_url" }, { "line": 591, "key": "catalog_number", "value_expression": "catalog_number" }, { "line": 592, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 593, "key": "formats", "value_expression": "','.join(formats)" }, { "line": 594, "key": "status", "value_expression": "status" }, { "line": 595, "key": "assets", "value_expression": "assets" }, { "line": 596, "key": "raw_data_json", "value_expression": "json.dumps({**raw_data, 'assets': assets}, ensure_ascii=False)" }, { "line": 597, "key": "raw_data", "value_expression": "raw_data" }, { "line": 598, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 596, "key": "assets", "value_expression": "assets" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._respectful_pause", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 1021, "line_end": 1024, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 1022, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 145, "line_end": 164, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_collection", "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "preflight_crawl" ], "literal_get_calls": [ { "line": 148, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 150, "receiver": "preflight", "key": "ok", "expression": "preflight.get('ok')" }, { "line": 151, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" }, { "line": 152, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" } ], "html_queries": [], "dictionary_entries": [ { "line": 149, "key": "collection", "value_expression": "','.join(collections)" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.preflight_crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 122, "line_end": 143, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 143, "key": "ok", "value_expression": "True" }, { "line": 125, "key": "ok", "value_expression": "False" }, { "line": 125, "key": "message", "value_expression": "'No supported digital sheet music collections were selected'" }, { "line": 140, "key": "ok", "value_expression": "False" }, { "line": 141, "key": "message", "value_expression": "'musescore_direct requires explicit URLs; an empty credential boundary is not a completed crawl'" }, { "line": 129, "key": "ok", "value_expression": "False" }, { "line": 129, "key": "message", "value_expression": "\"pdmx_mode must be either 'rows' or 'inventory'\"" }, { "line": 132, "key": "ok", "value_expression": "False" }, { "line": 134, "key": "message", "value_expression": "'PDMX row acquisition requires an existing pdmx_csv_path; use pdmx_mode=inventory only for an explicit dataset-inventory scope'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 166, "line_end": 179, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 167, "receiver": "raw_data", "key": "score_id", "expression": "raw_data.get('score_id')" }, { "line": 167, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 169, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 178, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "score_id", "value_expression": "raw_data.get('score_id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "id", "value_expression": "raw_data.get('id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 169, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "transform_run_id", "value_expression": "raw_data.get('transform_run_id')", "line": 178, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 148, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "ok", "value_expression": "preflight.get('ok')", "line": 150, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 172, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 174, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:\n pass\n return normalize_digital_sheet_music_record(" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 175, "line_end": 179, "text": " payload,\n primary_id=score_id,\n raw_content=json.dumps(payload, ensure_ascii=False, sort_keys=True),\n transform_run_id=raw_data.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 145, "line_end": 153, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 147, "line_end": 155, "text": " collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))\n records: List[Dict[str, Any]] = []\n for index, collection in enumerate(collections, start=1):" } ] } }, { "catalogue_id": "S057", "source_key": "score_mutopia", "name": "Mutopia Project", "selection_role": "score_collection", "parent_source_key": "digital_sheet_music", "attribution": "The Mutopia Project and individual typesetters", "scope_description": "General music collection containing organ works", "documentary_unit": "Typeset edition with editable and rendered files", "source_description": "Mutopia supplies PDF, MIDI and editable LilyPond files and credits the typesetter of each piece. Its licence page distinguishes public-domain contributions from several versions of Creative Commons Attribution and Attribution-ShareAlike licences.", "source_description_references": [ "score_mutopia", "mutopia_rights" ], "interpretation": "The edition and typesetter belong in the citation alongside the composer. The home-page music count is not an organ-work count. Keep the source file and generated performance or print format distinct.", "rights_observation": "Multiple licences apply by piece; the specific edition determines the obligations.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "score_mutopia", "mutopia_rights" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "This is the mutopia collection route inside DigitalSheetMusicCrawler, not a separately registered crawler. The source-specific listing parser reads tabular title, composer and catalogue-number cells, an instrumentation row, piece-information links and labelled assets. A separate continuation helper handles following listing pages.", "transformation": "The shared score normalizer separates work metadata, assets, formats, collection context and licence statements. The code recognises specific licence-link labels and asset labels. Unrecognised labels need review, and pagination support does not prove that a crawl reached every page. The editable LilyPond source, PDF and MIDI are different representations of a contribution.", "limitations": "The edition and typesetter belong in the citation alongside the composer. The home-page music count is not an organ-work count. Keep the source file and generated performance or print format distinct. Availability of a collection route does not demonstrate that all pages or files were acquired." }, "implementation": { "class": "DigitalSheetMusicCrawler", "class_line": 51, "path": "aggregator/digital_sheet_music_crawler.py", "inheritance": [ "DigitalSheetMusicCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "score_id", "path": "aggregator/digital_sheet_music_crawler.py", "line": 58, "class": "DigitalSheetMusicCrawler" }, "record_label": { "value": "digital scores", "path": "aggregator/digital_sheet_music_crawler.py", "line": 59, "class": "DigitalSheetMusicCrawler" } }, "methods": [ { "symbol": "DigitalSheetMusicCrawler._assets_from_html", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 857, "line_end": 905, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 865, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 864, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 887, "key": "url", "value_expression": "href" }, { "line": 887, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 887, "key": "format", "value_expression": "info['format']" }, { "line": 887, "key": "label", "value_expression": "text or info['format']" }, { "line": 897, "key": "url", "value_expression": "href" }, { "line": 898, "key": "alternate_urls", "value_expression": "_imslp_mirror_candidates(href, imslp_file_id)" }, { "line": 899, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 900, "key": "format", "value_expression": "info['format']" }, { "line": 901, "key": "label", "value_expression": "text or info['format']" }, { "line": 902, "key": "source_file_id", "value_expression": "imslp_file_id" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_collection", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 318, "line_end": 329, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_imslp_wima", "_crawl_musescore_direct", "_crawl_mutopia", "_crawl_orgelbuechlein", "_crawl_pdmx" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_imslp_wima", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 457, "line_end": 484, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 459, "receiver": "options", "key": "imslp_seed_urls", "expression": "options.get('imslp_seed_urls')" }, { "line": 460, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 461, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 479, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [], "dictionary_entries": [ { "line": 480, "key": "html_excerpt", "value_expression": "html[:10000]" }, { "line": 480, "key": "seed_url", "value_expression": "url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_musescore_direct", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 513, "line_end": 563, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 514, "receiver": "options", "key": "musescore_direct_urls", "expression": "options.get('musescore_direct_urls')" }, { "line": 515, "receiver": "self.config", "key": "musescore_credentials_env", "expression": "self.config.get('musescore_credentials_env')" }, { "line": 539, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 543, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 532, "key": "credential_required", "value_expression": "True" }, { "line": 533, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "credential_required", "value_expression": "True" }, { "line": 559, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "http_status", "value_expression": "response.status_code" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_mutopia", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 486, "line_end": 511, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_mutopia_next_url", "_parse_mutopia_listing", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 487, "receiver": "self.config", "key": "mutopia_collection_url", "expression": "self.config.get('mutopia_collection_url')" }, { "line": 488, "receiver": "self.config", "key": "mutopia_max_pages", "expression": "self.config.get('mutopia_max_pages', 0)" }, { "line": 496, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 500, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_options", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 280, "line_end": 316, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "source", "key": "collection", "expression": "source.get('collection')" }, { "line": 286, "receiver": "source", "key": "collections", "expression": "source.get('collections')" }, { "line": 293, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 295, "receiver": "source", "key": "organ_only", "expression": "source.get('organ_only')" }, { "line": 302, "receiver": "source", "key": "pdmx_enable_archive_streaming", "expression": "source.get('pdmx_enable_archive_streaming')" }, { "line": 288, "receiver": "self.config", "key": "default_collections", "expression": "self.config.get('default_collections')" }, { "line": 297, "receiver": "source", "key": "pdmx_csv_path", "expression": "source.get('pdmx_csv_path')" }, { "line": 297, "receiver": "self.config", "key": "pdmx_csv_path", "expression": "self.config.get('pdmx_csv_path')" }, { "line": 299, "receiver": "source", "key": "pdmx_root_dir", "expression": "source.get('pdmx_root_dir')" }, { "line": 299, "receiver": "self.config", "key": "pdmx_root_dir", "expression": "self.config.get('pdmx_root_dir')" }, { "line": 300, "receiver": "source", "key": "pdmx_zenodo_record_api_url", "expression": "source.get('pdmx_zenodo_record_api_url')" }, { "line": 300, "receiver": "self.config", "key": "pdmx_zenodo_record_api_url", "expression": "self.config.get('pdmx_zenodo_record_api_url')" }, { "line": 303, "receiver": "self.config", "key": "pdmx_enable_archive_streaming", "expression": "self.config.get('pdmx_enable_archive_streaming', False)" }, { "line": 306, "receiver": "source", "key": "pdmx_archive_stream_max_mb", "expression": "source.get('pdmx_archive_stream_max_mb')" }, { "line": 306, "receiver": "self.config", "key": "pdmx_archive_stream_max_mb", "expression": "self.config.get('pdmx_archive_stream_max_mb')" }, { "line": 314, "receiver": "source", "key": "imslp_seed_urls", "expression": "source.get('imslp_seed_urls')" }, { "line": 315, "receiver": "source", "key": "musescore_direct_urls", "expression": "source.get('musescore_direct_urls')" }, { "line": 310, "receiver": "source", "key": "pdmx_mxl_archive_url", "expression": "source.get('pdmx_mxl_archive_url')" }, { "line": 310, "receiver": "self.config", "key": "pdmx_mxl_archive_url", "expression": "self.config.get('pdmx_mxl_archive_url')" }, { "line": 311, "receiver": "source", "key": "pdmx_mid_archive_url", "expression": "source.get('pdmx_mid_archive_url')" }, { "line": 311, "receiver": "self.config", "key": "pdmx_mid_archive_url", "expression": "self.config.get('pdmx_mid_archive_url')" }, { "line": 312, "receiver": "source", "key": "pdmx_pdf_archive_url", "expression": "source.get('pdmx_pdf_archive_url')" }, { "line": 312, "receiver": "self.config", "key": "pdmx_pdf_archive_url", "expression": "self.config.get('pdmx_pdf_archive_url')" }, { "line": 314, "receiver": "self.config", "key": "imslp_seed_urls", "expression": "self.config.get('imslp_seed_urls')" }, { "line": 315, "receiver": "self.config", "key": "musescore_direct_urls", "expression": "self.config.get('musescore_direct_urls')" }, { "line": 294, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 296, "receiver": "source", "key": "license_filter", "expression": "source.get('license_filter')" }, { "line": 298, "receiver": "source", "key": "pdmx_mode", "expression": "source.get('pdmx_mode')" }, { "line": 298, "receiver": "self.config", "key": "pdmx_mode", "expression": "self.config.get('pdmx_mode')" } ], "html_queries": [], "dictionary_entries": [ { "line": 292, "key": "collections", "value_expression": "[collection for collection in collections if collection in SOURCE_COLLECTIONS]" }, { "line": 293, "key": "limit", "value_expression": "_safe_int(source.get('limit'), default=0)" }, { "line": 294, "key": "query", "value_expression": "str(source.get('query') or '').strip().lower()" }, { "line": 295, "key": "organ_only", "value_expression": "_read_bool(source.get('organ_only'), True)" }, { "line": 296, "key": "license_filter", "value_expression": "str(source.get('license_filter') or '').strip().lower()" }, { "line": 297, "key": "pdmx_csv_path", "value_expression": "str(source.get('pdmx_csv_path') or self.config.get('pdmx_csv_path') or '')" }, { "line": 298, "key": "pdmx_mode", "value_expression": "str(source.get('pdmx_mode') or self.config.get('pdmx_mode') or 'rows').strip().lower()" }, { "line": 299, "key": "pdmx_root_dir", "value_expression": "str(source.get('pdmx_root_dir') or self.config.get('pdmx_root_dir') or '')" }, { "line": 300, "key": "pdmx_zenodo_record_api_url", "value_expression": "str(source.get('pdmx_zenodo_record_api_url') or self.config.get('pdmx_zenodo_record_api_url') or '')" }, { "line": 301, "key": "pdmx_enable_archive_streaming", "value_expression": "_read_bool(source.get('pdmx_enable_archive_streaming'), bool(self.config.get('pdmx_enable_archive_streaming', False)))" }, { "line": 305, "key": "pdmx_archive_stream_max_mb", "value_expression": "_safe_int(source.get('pdmx_archive_stream_max_mb') or self.config.get('pdmx_archive_stream_max_mb'), default=256)" }, { "line": 309, "key": "pdmx_archive_urls", "value_expression": "{'mxl': str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or ''), 'mid': str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or ''), 'pdf': str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')}" }, { "line": 314, "key": "imslp_seed_urls", "value_expression": "_split_lines(source.get('imslp_seed_urls')) or list(self.config.get('imslp_seed_urls') or [])" }, { "line": 315, "key": "musescore_direct_urls", "value_expression": "_split_lines(source.get('musescore_direct_urls')) or list(self.config.get('musescore_direct_urls') or [])" }, { "line": 310, "key": "mxl", "value_expression": "str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or '')" }, { "line": 311, "key": "mid", "value_expression": "str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or '')" }, { "line": 312, "key": "pdf", "value_expression": "str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')" } ], "subscript_assignments": [ { "line": 285, "value_line": 285, "key": "collection", "target_expression": "source['collection']", "value_expression": "str(target)" } ] }, { "symbol": "DigitalSheetMusicCrawler._crawl_orgelbuechlein", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 416, "line_end": 455, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 417, "receiver": "self.config", "key": "orgelbuechlein_api_url", "expression": "self.config.get('orgelbuechlein_api_url')" }, { "line": 429, "receiver": "item", "key": "download_url", "expression": "item.get('download_url')" }, { "line": 429, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 418, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 423, "receiver": "item", "key": "name", "expression": "item.get('name')" }, { "line": 436, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 453, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "github_item", "value_expression": "item" }, { "line": 442, "key": "url", "value_expression": "raw_url" }, { "line": 442, "key": "file_name", "value_expression": "name" }, { "line": 442, "key": "format", "value_expression": "asset_format_from_url(name)['format']" }, { "line": 442, "key": "label", "value_expression": "'MusicXML'" }, { "line": 444, "key": "url", "value_expression": "raw_url.replace('/little-organ-book-mxl/', '/little-organ-book-pdf/').rsplit('.', 1)[0] + '.pdf'" }, { "line": 445, "key": "file_name", "value_expression": "f'{stem}.pdf'" }, { "line": 446, "key": "format", "value_expression": "'pdf'" }, { "line": 447, "key": "label", "value_expression": "'PDF'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 331, "line_end": 364, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_pdmx_zenodo_inventory", "_pdmx_assets", "_pdmx_row_matches", "_raw_record", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 332, "receiver": "options", "key": "pdmx_mode", "expression": "options.get('pdmx_mode')" }, { "line": 338, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 334, "receiver": "options", "key": "pdmx_csv_path", "expression": "options.get('pdmx_csv_path')" }, { "line": 337, "receiver": "options", "key": "pdmx_root_dir", "expression": "options.get('pdmx_root_dir')" }, { "line": 338, "receiver": "self.config", "key": "pdmx_max_rows", "expression": "self.config.get('pdmx_max_rows')" }, { "line": 348, "receiver": "row", "key": "path", "expression": "row.get('path')" }, { "line": 348, "receiver": "row", "key": "mxl", "expression": "row.get('mxl')" }, { "line": 348, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "song_name", "expression": "row.get('song_name')" }, { "line": 354, "receiver": "row", "key": "composer_name", "expression": "row.get('composer_name')" }, { "line": 354, "receiver": "row", "key": "artist_name", "expression": "row.get('artist_name')" }, { "line": 356, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 357, "receiver": "row", "key": "license_url", "expression": "row.get('license_url')" }, { "line": 359, "receiver": "row", "key": "instrumentation", "expression": "row.get('instrumentation')" }, { "line": 359, "receiver": "row", "key": "instruments", "expression": "row.get('instruments')" }, { "line": 359, "receiver": "row", "key": "instrument", "expression": "row.get('instrument')" } ], "html_queries": [], "dictionary_entries": [ { "line": 361, "key": "pdmx", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx_zenodo_inventory", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 366, "line_end": 414, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 373, "receiver": "payload", "key": "files", "expression": "payload.get('files')" }, { "line": 390, "receiver": "payload.get('metadata', {})", "key": "title", "expression": "payload.get('metadata', {}).get('title')" }, { "line": 367, "receiver": "options", "key": "pdmx_zenodo_record_api_url", "expression": "options.get('pdmx_zenodo_record_api_url')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "content", "expression": "item.get('links', {}).get('content')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "self", "expression": "item.get('links', {}).get('self')" }, { "line": 370, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 376, "receiver": "item", "key": "key", "expression": "item.get('key')" }, { "line": 386, "receiver": "item", "key": "size", "expression": "item.get('size')" }, { "line": 390, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 397, "receiver": "payload.get('links', {})", "key": "html", "expression": "payload.get('links', {}).get('html')" }, { "line": 398, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "id", "expression": "payload.get('metadata', {}).get('license', {}).get('id')" }, { "line": 399, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "url", "expression": "payload.get('metadata', {}).get('license', {}).get('url')" }, { "line": 406, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 407, "receiver": "payload", "key": "doi", "expression": "payload.get('doi')" }, { "line": 393, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 397, "receiver": "payload", "key": "links", "expression": "payload.get('links', {})" }, { "line": 398, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 399, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 398, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 399, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 382, "key": "url", "value_expression": "content_url" }, { "line": 383, "key": "file_name", "value_expression": "key" }, { "line": 384, "key": "format", "value_expression": "asset_format_from_url(key)['format']" }, { "line": 385, "key": "label", "value_expression": "'Zenodo dataset file'" }, { "line": 386, "key": "content_size_bytes", "value_expression": "item.get('size')" }, { "line": 387, "key": "download_status", "value_expression": "'dataset_snapshot_required'" }, { "line": 405, "key": "zenodo_record", "value_expression": "{'id': payload.get('id'), 'doi': payload.get('doi'), 'api_url': api_url, 'files': files}" }, { "line": 411, "key": "access_note", "value_expression": "'PDMX requires an explicit local snapshot path for score-level crawling to avoid automatic multi-GB dataset downloads.'" }, { "line": 406, "key": "id", "value_expression": "payload.get('id')" }, { "line": 407, "key": "doi", "value_expression": "payload.get('doi')" }, { "line": 408, "key": "api_url", "value_expression": "api_url" }, { "line": 409, "key": "files", "value_expression": "files" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._extract_tar_gz_member_from_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 780, "line_end": 812, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 785, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._mutopia_next_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 990, "line_end": 995, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 994, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 992, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._parse_mutopia_listing", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 907, "line_end": 988, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 939, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 951, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 912, "receiver": "row", "key": "class", "expression": "row.get('class')" }, { "line": 980, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [ { "line": 911, "expression": "soup.select('table.result-table')" }, { "line": 915, "expression": "rows[0].find_all('td', recursive=False)" }, { "line": 937, "expression": "table.find_all('a', href=True)" }, { "line": 947, "expression": "table.find_all('a', href=True)" }, { "line": 928, "expression": "rows[1].find_all('td', recursive=False)" }, { "line": 912, "expression": "table.find_all('tr', recursive=False)" } ], "dictionary_entries": [ { "line": 972, "key": "source_collection", "value_expression": "'mutopia'" }, { "line": 973, "key": "title", "value_expression": "title" }, { "line": 974, "key": "composer", "value_expression": "composer" }, { "line": 975, "key": "source_url", "value_expression": "piece_url or identity_url" }, { "line": 976, "key": "license_name", "value_expression": "license_name" }, { "line": 977, "key": "license_url", "value_expression": "license_url" }, { "line": 978, "key": "catalog_number", "value_expression": "catalog_number or _extract_bwv(title)" }, { "line": 979, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 980, "key": "assets", "value_expression": "_limited(assets, int(self.config.get('max_assets_per_record', 0) or 0))" }, { "line": 981, "key": "raw_data", "value_expression": "{'mutopia_listing_title': title, 'mutopia_piece_url': piece_url, 'mutopia_listing_url': source_url}" }, { "line": 954, "key": "url", "value_expression": "href" }, { "line": 955, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 956, "key": "format", "value_expression": "asset_format_from_url(href)['format']" }, { "line": 957, "key": "label", "value_expression": "label" }, { "line": 982, "key": "mutopia_listing_title", "value_expression": "title" }, { "line": 983, "key": "mutopia_piece_url", "value_expression": "piece_url" }, { "line": 984, "key": "mutopia_listing_url", "value_expression": "source_url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_assets", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 832, "line_end": 855, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 835, "receiver": "options", "key": "pdmx_enable_archive_streaming", "expression": "options.get('pdmx_enable_archive_streaming')" }, { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 850, "receiver": "options", "key": "pdmx_archive_stream_max_mb", "expression": "options.get('pdmx_archive_stream_max_mb')" } ], "html_queries": [], "dictionary_entries": [ { "line": 841, "key": "file_name", "value_expression": "Path(value).name" }, { "line": 841, "key": "format", "value_expression": "asset_format_from_url(value)['format']" }, { "line": 841, "key": "label", "value_expression": "label" } ], "subscript_assignments": [ { "line": 843, "value_line": 843, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 845, "value_line": 845, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 848, "value_line": 848, "key": "archive_url", "target_expression": "asset['archive_url']", "value_expression": "archive_url" }, { "line": 849, "value_line": 849, "key": "archive_member_path", "target_expression": "asset['archive_member_path']", "value_expression": "value.replace('./', '', 1)" }, { "line": 850, "value_line": 850, "key": "archive_stream_max_mb", "target_expression": "asset['archive_stream_max_mb']", "value_expression": "options.get('pdmx_archive_stream_max_mb')" }, { "line": 851, "value_line": 851, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'archive_stream_required'" }, { "line": 853, "value_line": 853, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'local_snapshot_missing'" } ] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_row_matches", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 814, "line_end": 830, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 821, "receiver": "options", "key": "license_filter", "expression": "options.get('license_filter')" }, { "line": 824, "receiver": "options", "key": "query", "expression": "options.get('query')" }, { "line": 826, "receiver": "options", "key": "organ_only", "expression": "options.get('organ_only')" }, { "line": 827, "receiver": "self.config", "key": "pdmx_include_track_19", "expression": "self.config.get('pdmx_include_track_19', True)" }, { "line": 815, "receiver": "row", "key": "subset:deduplicated", "expression": "row.get('subset:deduplicated', '')" }, { "line": 817, "receiver": "row", "key": "subset:no_license_conflict", "expression": "row.get('subset:no_license_conflict', '')" }, { "line": 819, "receiver": "row", "key": "subset:all_valid", "expression": "row.get('subset:all_valid', '')" }, { "line": 821, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 827, "receiver": "row", "key": "tracks", "expression": "row.get('tracks')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._raw_record", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 565, "line_end": 599, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 581, "receiver": "asset", "key": "format", "expression": "asset.get('format')" }, { "line": 581, "receiver": "asset", "key": "url", "expression": "asset.get('url')" }, { "line": 581, "receiver": "asset", "key": "local_path", "expression": "asset.get('local_path')" } ], "html_queries": [], "dictionary_entries": [ { "line": 583, "key": "score_id", "value_expression": "score_id" }, { "line": 584, "key": "source_collection", "value_expression": "source_collection" }, { "line": 585, "key": "title", "value_expression": "title" }, { "line": 586, "key": "composer", "value_expression": "composer" }, { "line": 587, "key": "source_url", "value_expression": "source_url" }, { "line": 588, "key": "url", "value_expression": "source_url" }, { "line": 589, "key": "license_name", "value_expression": "license_name" }, { "line": 590, "key": "license_url", "value_expression": "license_url" }, { "line": 591, "key": "catalog_number", "value_expression": "catalog_number" }, { "line": 592, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 593, "key": "formats", "value_expression": "','.join(formats)" }, { "line": 594, "key": "status", "value_expression": "status" }, { "line": 595, "key": "assets", "value_expression": "assets" }, { "line": 596, "key": "raw_data_json", "value_expression": "json.dumps({**raw_data, 'assets': assets}, ensure_ascii=False)" }, { "line": 597, "key": "raw_data", "value_expression": "raw_data" }, { "line": 598, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 596, "key": "assets", "value_expression": "assets" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._respectful_pause", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 1021, "line_end": 1024, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 1022, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 145, "line_end": 164, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_collection", "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "preflight_crawl" ], "literal_get_calls": [ { "line": 148, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 150, "receiver": "preflight", "key": "ok", "expression": "preflight.get('ok')" }, { "line": 151, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" }, { "line": 152, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" } ], "html_queries": [], "dictionary_entries": [ { "line": 149, "key": "collection", "value_expression": "','.join(collections)" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.preflight_crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 122, "line_end": 143, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 143, "key": "ok", "value_expression": "True" }, { "line": 125, "key": "ok", "value_expression": "False" }, { "line": 125, "key": "message", "value_expression": "'No supported digital sheet music collections were selected'" }, { "line": 140, "key": "ok", "value_expression": "False" }, { "line": 141, "key": "message", "value_expression": "'musescore_direct requires explicit URLs; an empty credential boundary is not a completed crawl'" }, { "line": 129, "key": "ok", "value_expression": "False" }, { "line": 129, "key": "message", "value_expression": "\"pdmx_mode must be either 'rows' or 'inventory'\"" }, { "line": 132, "key": "ok", "value_expression": "False" }, { "line": 134, "key": "message", "value_expression": "'PDMX row acquisition requires an existing pdmx_csv_path; use pdmx_mode=inventory only for an explicit dataset-inventory scope'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 166, "line_end": 179, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 167, "receiver": "raw_data", "key": "score_id", "expression": "raw_data.get('score_id')" }, { "line": 167, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 169, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 178, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "score_id", "value_expression": "raw_data.get('score_id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "id", "value_expression": "raw_data.get('id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 169, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "transform_run_id", "value_expression": "raw_data.get('transform_run_id')", "line": 178, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 148, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "ok", "value_expression": "preflight.get('ok')", "line": 150, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 172, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 174, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:\n pass\n return normalize_digital_sheet_music_record(" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 175, "line_end": 179, "text": " payload,\n primary_id=score_id,\n raw_content=json.dumps(payload, ensure_ascii=False, sort_keys=True),\n transform_run_id=raw_data.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 145, "line_end": 153, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 147, "line_end": 155, "text": " collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))\n records: List[Dict[str, Any]] = []\n for index, collection in enumerate(collections, start=1):" } ] } }, { "catalogue_id": "S058", "source_key": "score_musescore_direct", "name": "MuseScore.com Direct", "selection_role": "score_collection", "parent_source_key": "digital_sheet_music", "attribution": "MuseScore.com and individual score contributors", "scope_description": "Directly selected score URLs", "documentary_unit": "User-published score version", "source_description": "The local collection definition requires credentials and accepts explicit MuseScore URLs. The public entry-point check encountered an access challenge; no score content or current download terms were verified.", "source_description_references": [ "score_musescore_direct", "repo_score_collections" ], "interpretation": "Record contributor, score identifier, revision and downloaded format for any later acquisition. Platform identity does not identify a score's editor, and a public-domain composition does not determine the rights in its modern encoding.", "rights_observation": "File-specific permissions and current account conditions remain unverified.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "score_musescore_direct", "repo_score_collections" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "code_reading_and_interpretation", "acquisition": "This is the musescore_direct collection route inside DigitalSheetMusicCrawler, not a separately registered crawler. The collection definition marks the direct route as requiring credentials and the implementation accepts explicitly selected score addresses.", "transformation": "The shared score normalizer separates work metadata, assets, formats, collection context and licence statements. The dated source check encountered an access challenge. No source-content inspection, current download entitlement or successful acquisition is established by the configured route.", "limitations": "Record contributor, score identifier, revision and downloaded format for any later acquisition. Platform identity does not identify a score's editor, and a public-domain composition does not determine the rights in its modern encoding. Availability of a collection route does not demonstrate that all pages or files were acquired." }, "implementation": { "class": "DigitalSheetMusicCrawler", "class_line": 51, "path": "aggregator/digital_sheet_music_crawler.py", "inheritance": [ "DigitalSheetMusicCrawler", "BaseCrawler" ], "configuration": { "transformed_json_id_column": { "value": "score_id", "path": "aggregator/digital_sheet_music_crawler.py", "line": 58, "class": "DigitalSheetMusicCrawler" }, "record_label": { "value": "digital scores", "path": "aggregator/digital_sheet_music_crawler.py", "line": 59, "class": "DigitalSheetMusicCrawler" } }, "methods": [ { "symbol": "DigitalSheetMusicCrawler._assets_from_html", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 857, "line_end": 905, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 865, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 864, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [ { "line": 887, "key": "url", "value_expression": "href" }, { "line": 887, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 887, "key": "format", "value_expression": "info['format']" }, { "line": 887, "key": "label", "value_expression": "text or info['format']" }, { "line": 897, "key": "url", "value_expression": "href" }, { "line": 898, "key": "alternate_urls", "value_expression": "_imslp_mirror_candidates(href, imslp_file_id)" }, { "line": 899, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 900, "key": "format", "value_expression": "info['format']" }, { "line": 901, "key": "label", "value_expression": "text or info['format']" }, { "line": 902, "key": "source_file_id", "value_expression": "imslp_file_id" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_collection", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 318, "line_end": 329, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_imslp_wima", "_crawl_musescore_direct", "_crawl_mutopia", "_crawl_orgelbuechlein", "_crawl_pdmx" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_imslp_wima", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 457, "line_end": 484, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 459, "receiver": "options", "key": "imslp_seed_urls", "expression": "options.get('imslp_seed_urls')" }, { "line": 460, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 461, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 479, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [], "dictionary_entries": [ { "line": 480, "key": "html_excerpt", "value_expression": "html[:10000]" }, { "line": 480, "key": "seed_url", "value_expression": "url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_musescore_direct", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 513, "line_end": 563, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_assets_from_html", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 514, "receiver": "options", "key": "musescore_direct_urls", "expression": "options.get('musescore_direct_urls')" }, { "line": 515, "receiver": "self.config", "key": "musescore_credentials_env", "expression": "self.config.get('musescore_credentials_env')" }, { "line": 539, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 543, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [ { "line": 532, "key": "credential_required", "value_expression": "True" }, { "line": 533, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "credential_required", "value_expression": "True" }, { "line": 559, "key": "credential_available", "value_expression": "credential_available" }, { "line": 559, "key": "http_status", "value_expression": "response.status_code" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_mutopia", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 486, "line_end": 511, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_mutopia_next_url", "_parse_mutopia_listing", "_raw_record", "_respectful_pause" ], "literal_get_calls": [ { "line": 487, "receiver": "self.config", "key": "mutopia_collection_url", "expression": "self.config.get('mutopia_collection_url')" }, { "line": 488, "receiver": "self.config", "key": "mutopia_max_pages", "expression": "self.config.get('mutopia_max_pages', 0)" }, { "line": 496, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 500, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_options", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 280, "line_end": 316, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 286, "receiver": "source", "key": "collection", "expression": "source.get('collection')" }, { "line": 286, "receiver": "source", "key": "collections", "expression": "source.get('collections')" }, { "line": 293, "receiver": "source", "key": "limit", "expression": "source.get('limit')" }, { "line": 295, "receiver": "source", "key": "organ_only", "expression": "source.get('organ_only')" }, { "line": 302, "receiver": "source", "key": "pdmx_enable_archive_streaming", "expression": "source.get('pdmx_enable_archive_streaming')" }, { "line": 288, "receiver": "self.config", "key": "default_collections", "expression": "self.config.get('default_collections')" }, { "line": 297, "receiver": "source", "key": "pdmx_csv_path", "expression": "source.get('pdmx_csv_path')" }, { "line": 297, "receiver": "self.config", "key": "pdmx_csv_path", "expression": "self.config.get('pdmx_csv_path')" }, { "line": 299, "receiver": "source", "key": "pdmx_root_dir", "expression": "source.get('pdmx_root_dir')" }, { "line": 299, "receiver": "self.config", "key": "pdmx_root_dir", "expression": "self.config.get('pdmx_root_dir')" }, { "line": 300, "receiver": "source", "key": "pdmx_zenodo_record_api_url", "expression": "source.get('pdmx_zenodo_record_api_url')" }, { "line": 300, "receiver": "self.config", "key": "pdmx_zenodo_record_api_url", "expression": "self.config.get('pdmx_zenodo_record_api_url')" }, { "line": 303, "receiver": "self.config", "key": "pdmx_enable_archive_streaming", "expression": "self.config.get('pdmx_enable_archive_streaming', False)" }, { "line": 306, "receiver": "source", "key": "pdmx_archive_stream_max_mb", "expression": "source.get('pdmx_archive_stream_max_mb')" }, { "line": 306, "receiver": "self.config", "key": "pdmx_archive_stream_max_mb", "expression": "self.config.get('pdmx_archive_stream_max_mb')" }, { "line": 314, "receiver": "source", "key": "imslp_seed_urls", "expression": "source.get('imslp_seed_urls')" }, { "line": 315, "receiver": "source", "key": "musescore_direct_urls", "expression": "source.get('musescore_direct_urls')" }, { "line": 310, "receiver": "source", "key": "pdmx_mxl_archive_url", "expression": "source.get('pdmx_mxl_archive_url')" }, { "line": 310, "receiver": "self.config", "key": "pdmx_mxl_archive_url", "expression": "self.config.get('pdmx_mxl_archive_url')" }, { "line": 311, "receiver": "source", "key": "pdmx_mid_archive_url", "expression": "source.get('pdmx_mid_archive_url')" }, { "line": 311, "receiver": "self.config", "key": "pdmx_mid_archive_url", "expression": "self.config.get('pdmx_mid_archive_url')" }, { "line": 312, "receiver": "source", "key": "pdmx_pdf_archive_url", "expression": "source.get('pdmx_pdf_archive_url')" }, { "line": 312, "receiver": "self.config", "key": "pdmx_pdf_archive_url", "expression": "self.config.get('pdmx_pdf_archive_url')" }, { "line": 314, "receiver": "self.config", "key": "imslp_seed_urls", "expression": "self.config.get('imslp_seed_urls')" }, { "line": 315, "receiver": "self.config", "key": "musescore_direct_urls", "expression": "self.config.get('musescore_direct_urls')" }, { "line": 294, "receiver": "source", "key": "query", "expression": "source.get('query')" }, { "line": 296, "receiver": "source", "key": "license_filter", "expression": "source.get('license_filter')" }, { "line": 298, "receiver": "source", "key": "pdmx_mode", "expression": "source.get('pdmx_mode')" }, { "line": 298, "receiver": "self.config", "key": "pdmx_mode", "expression": "self.config.get('pdmx_mode')" } ], "html_queries": [], "dictionary_entries": [ { "line": 292, "key": "collections", "value_expression": "[collection for collection in collections if collection in SOURCE_COLLECTIONS]" }, { "line": 293, "key": "limit", "value_expression": "_safe_int(source.get('limit'), default=0)" }, { "line": 294, "key": "query", "value_expression": "str(source.get('query') or '').strip().lower()" }, { "line": 295, "key": "organ_only", "value_expression": "_read_bool(source.get('organ_only'), True)" }, { "line": 296, "key": "license_filter", "value_expression": "str(source.get('license_filter') or '').strip().lower()" }, { "line": 297, "key": "pdmx_csv_path", "value_expression": "str(source.get('pdmx_csv_path') or self.config.get('pdmx_csv_path') or '')" }, { "line": 298, "key": "pdmx_mode", "value_expression": "str(source.get('pdmx_mode') or self.config.get('pdmx_mode') or 'rows').strip().lower()" }, { "line": 299, "key": "pdmx_root_dir", "value_expression": "str(source.get('pdmx_root_dir') or self.config.get('pdmx_root_dir') or '')" }, { "line": 300, "key": "pdmx_zenodo_record_api_url", "value_expression": "str(source.get('pdmx_zenodo_record_api_url') or self.config.get('pdmx_zenodo_record_api_url') or '')" }, { "line": 301, "key": "pdmx_enable_archive_streaming", "value_expression": "_read_bool(source.get('pdmx_enable_archive_streaming'), bool(self.config.get('pdmx_enable_archive_streaming', False)))" }, { "line": 305, "key": "pdmx_archive_stream_max_mb", "value_expression": "_safe_int(source.get('pdmx_archive_stream_max_mb') or self.config.get('pdmx_archive_stream_max_mb'), default=256)" }, { "line": 309, "key": "pdmx_archive_urls", "value_expression": "{'mxl': str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or ''), 'mid': str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or ''), 'pdf': str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')}" }, { "line": 314, "key": "imslp_seed_urls", "value_expression": "_split_lines(source.get('imslp_seed_urls')) or list(self.config.get('imslp_seed_urls') or [])" }, { "line": 315, "key": "musescore_direct_urls", "value_expression": "_split_lines(source.get('musescore_direct_urls')) or list(self.config.get('musescore_direct_urls') or [])" }, { "line": 310, "key": "mxl", "value_expression": "str(source.get('pdmx_mxl_archive_url') or self.config.get('pdmx_mxl_archive_url') or '')" }, { "line": 311, "key": "mid", "value_expression": "str(source.get('pdmx_mid_archive_url') or self.config.get('pdmx_mid_archive_url') or '')" }, { "line": 312, "key": "pdf", "value_expression": "str(source.get('pdmx_pdf_archive_url') or self.config.get('pdmx_pdf_archive_url') or '')" } ], "subscript_assignments": [ { "line": 285, "value_line": 285, "key": "collection", "target_expression": "source['collection']", "value_expression": "str(target)" } ] }, { "symbol": "DigitalSheetMusicCrawler._crawl_orgelbuechlein", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 416, "line_end": 455, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 417, "receiver": "self.config", "key": "orgelbuechlein_api_url", "expression": "self.config.get('orgelbuechlein_api_url')" }, { "line": 429, "receiver": "item", "key": "download_url", "expression": "item.get('download_url')" }, { "line": 429, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 418, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 423, "receiver": "item", "key": "name", "expression": "item.get('name')" }, { "line": 436, "receiver": "item", "key": "html_url", "expression": "item.get('html_url')" }, { "line": 453, "receiver": "options", "key": "limit", "expression": "options.get('limit')" } ], "html_queries": [], "dictionary_entries": [ { "line": 450, "key": "github_item", "value_expression": "item" }, { "line": 442, "key": "url", "value_expression": "raw_url" }, { "line": 442, "key": "file_name", "value_expression": "name" }, { "line": 442, "key": "format", "value_expression": "asset_format_from_url(name)['format']" }, { "line": 442, "key": "label", "value_expression": "'MusicXML'" }, { "line": 444, "key": "url", "value_expression": "raw_url.replace('/little-organ-book-mxl/', '/little-organ-book-pdf/').rsplit('.', 1)[0] + '.pdf'" }, { "line": 445, "key": "file_name", "value_expression": "f'{stem}.pdf'" }, { "line": 446, "key": "format", "value_expression": "'pdf'" }, { "line": 447, "key": "label", "value_expression": "'PDF'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 331, "line_end": 364, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_pdmx_zenodo_inventory", "_pdmx_assets", "_pdmx_row_matches", "_raw_record", "authorize_source_file_io" ], "literal_get_calls": [ { "line": 332, "receiver": "options", "key": "pdmx_mode", "expression": "options.get('pdmx_mode')" }, { "line": 338, "receiver": "options", "key": "limit", "expression": "options.get('limit')" }, { "line": 334, "receiver": "options", "key": "pdmx_csv_path", "expression": "options.get('pdmx_csv_path')" }, { "line": 337, "receiver": "options", "key": "pdmx_root_dir", "expression": "options.get('pdmx_root_dir')" }, { "line": 338, "receiver": "self.config", "key": "pdmx_max_rows", "expression": "self.config.get('pdmx_max_rows')" }, { "line": 348, "receiver": "row", "key": "path", "expression": "row.get('path')" }, { "line": 348, "receiver": "row", "key": "mxl", "expression": "row.get('mxl')" }, { "line": 348, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "title", "expression": "row.get('title')" }, { "line": 353, "receiver": "row", "key": "song_name", "expression": "row.get('song_name')" }, { "line": 354, "receiver": "row", "key": "composer_name", "expression": "row.get('composer_name')" }, { "line": 354, "receiver": "row", "key": "artist_name", "expression": "row.get('artist_name')" }, { "line": 356, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 357, "receiver": "row", "key": "license_url", "expression": "row.get('license_url')" }, { "line": 359, "receiver": "row", "key": "instrumentation", "expression": "row.get('instrumentation')" }, { "line": 359, "receiver": "row", "key": "instruments", "expression": "row.get('instruments')" }, { "line": 359, "receiver": "row", "key": "instrument", "expression": "row.get('instrument')" } ], "html_queries": [], "dictionary_entries": [ { "line": 361, "key": "pdmx", "value_expression": "row" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._crawl_pdmx_zenodo_inventory", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 366, "line_end": 414, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_raw_record" ], "literal_get_calls": [ { "line": 373, "receiver": "payload", "key": "files", "expression": "payload.get('files')" }, { "line": 390, "receiver": "payload.get('metadata', {})", "key": "title", "expression": "payload.get('metadata', {}).get('title')" }, { "line": 367, "receiver": "options", "key": "pdmx_zenodo_record_api_url", "expression": "options.get('pdmx_zenodo_record_api_url')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "content", "expression": "item.get('links', {}).get('content')" }, { "line": 377, "receiver": "item.get('links', {})", "key": "self", "expression": "item.get('links', {}).get('self')" }, { "line": 370, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" }, { "line": 376, "receiver": "item", "key": "key", "expression": "item.get('key')" }, { "line": 386, "receiver": "item", "key": "size", "expression": "item.get('size')" }, { "line": 390, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 377, "receiver": "item", "key": "links", "expression": "item.get('links', {})" }, { "line": 397, "receiver": "payload.get('links', {})", "key": "html", "expression": "payload.get('links', {}).get('html')" }, { "line": 398, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "id", "expression": "payload.get('metadata', {}).get('license', {}).get('id')" }, { "line": 399, "receiver": "payload.get('metadata', {}).get('license', {})", "key": "url", "expression": "payload.get('metadata', {}).get('license', {}).get('url')" }, { "line": 406, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 407, "receiver": "payload", "key": "doi", "expression": "payload.get('doi')" }, { "line": 393, "receiver": "payload", "key": "id", "expression": "payload.get('id')" }, { "line": 397, "receiver": "payload", "key": "links", "expression": "payload.get('links', {})" }, { "line": 398, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 399, "receiver": "payload.get('metadata', {})", "key": "license", "expression": "payload.get('metadata', {}).get('license', {})" }, { "line": 398, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" }, { "line": 399, "receiver": "payload", "key": "metadata", "expression": "payload.get('metadata', {})" } ], "html_queries": [], "dictionary_entries": [ { "line": 382, "key": "url", "value_expression": "content_url" }, { "line": 383, "key": "file_name", "value_expression": "key" }, { "line": 384, "key": "format", "value_expression": "asset_format_from_url(key)['format']" }, { "line": 385, "key": "label", "value_expression": "'Zenodo dataset file'" }, { "line": 386, "key": "content_size_bytes", "value_expression": "item.get('size')" }, { "line": 387, "key": "download_status", "value_expression": "'dataset_snapshot_required'" }, { "line": 405, "key": "zenodo_record", "value_expression": "{'id': payload.get('id'), 'doi': payload.get('doi'), 'api_url': api_url, 'files': files}" }, { "line": 411, "key": "access_note", "value_expression": "'PDMX requires an explicit local snapshot path for score-level crawling to avoid automatic multi-GB dataset downloads.'" }, { "line": 406, "key": "id", "value_expression": "payload.get('id')" }, { "line": 407, "key": "doi", "value_expression": "payload.get('doi')" }, { "line": 408, "key": "api_url", "value_expression": "api_url" }, { "line": 409, "key": "files", "value_expression": "files" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._extract_tar_gz_member_from_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 780, "line_end": 812, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 785, "receiver": "self.config", "key": "timeout", "expression": "self.config.get('timeout', 30)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._mutopia_next_url", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 990, "line_end": 995, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 994, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" } ], "html_queries": [ { "line": 992, "expression": "soup.find_all('a', href=True)" } ], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._parse_mutopia_listing", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 907, "line_end": 988, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 939, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 951, "receiver": "anchor", "key": "href", "expression": "anchor.get('href')" }, { "line": 912, "receiver": "row", "key": "class", "expression": "row.get('class')" }, { "line": 980, "receiver": "self.config", "key": "max_assets_per_record", "expression": "self.config.get('max_assets_per_record', 0)" } ], "html_queries": [ { "line": 911, "expression": "soup.select('table.result-table')" }, { "line": 915, "expression": "rows[0].find_all('td', recursive=False)" }, { "line": 937, "expression": "table.find_all('a', href=True)" }, { "line": 947, "expression": "table.find_all('a', href=True)" }, { "line": 928, "expression": "rows[1].find_all('td', recursive=False)" }, { "line": 912, "expression": "table.find_all('tr', recursive=False)" } ], "dictionary_entries": [ { "line": 972, "key": "source_collection", "value_expression": "'mutopia'" }, { "line": 973, "key": "title", "value_expression": "title" }, { "line": 974, "key": "composer", "value_expression": "composer" }, { "line": 975, "key": "source_url", "value_expression": "piece_url or identity_url" }, { "line": 976, "key": "license_name", "value_expression": "license_name" }, { "line": 977, "key": "license_url", "value_expression": "license_url" }, { "line": 978, "key": "catalog_number", "value_expression": "catalog_number or _extract_bwv(title)" }, { "line": 979, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 980, "key": "assets", "value_expression": "_limited(assets, int(self.config.get('max_assets_per_record', 0) or 0))" }, { "line": 981, "key": "raw_data", "value_expression": "{'mutopia_listing_title': title, 'mutopia_piece_url': piece_url, 'mutopia_listing_url': source_url}" }, { "line": 954, "key": "url", "value_expression": "href" }, { "line": 955, "key": "file_name", "value_expression": "os.path.basename(urlparse(href).path)" }, { "line": 956, "key": "format", "value_expression": "asset_format_from_url(href)['format']" }, { "line": 957, "key": "label", "value_expression": "label" }, { "line": 982, "key": "mutopia_listing_title", "value_expression": "title" }, { "line": 983, "key": "mutopia_piece_url", "value_expression": "piece_url" }, { "line": 984, "key": "mutopia_listing_url", "value_expression": "source_url" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_assets", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 832, "line_end": 855, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 835, "receiver": "options", "key": "pdmx_enable_archive_streaming", "expression": "options.get('pdmx_enable_archive_streaming')" }, { "line": 834, "receiver": "options", "key": "pdmx_archive_urls", "expression": "options.get('pdmx_archive_urls')" }, { "line": 850, "receiver": "options", "key": "pdmx_archive_stream_max_mb", "expression": "options.get('pdmx_archive_stream_max_mb')" } ], "html_queries": [], "dictionary_entries": [ { "line": 841, "key": "file_name", "value_expression": "Path(value).name" }, { "line": 841, "key": "format", "value_expression": "asset_format_from_url(value)['format']" }, { "line": 841, "key": "label", "value_expression": "label" } ], "subscript_assignments": [ { "line": 843, "value_line": 843, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 845, "value_line": 845, "key": "local_path", "target_expression": "asset['local_path']", "value_expression": "str(local_path)" }, { "line": 848, "value_line": 848, "key": "archive_url", "target_expression": "asset['archive_url']", "value_expression": "archive_url" }, { "line": 849, "value_line": 849, "key": "archive_member_path", "target_expression": "asset['archive_member_path']", "value_expression": "value.replace('./', '', 1)" }, { "line": 850, "value_line": 850, "key": "archive_stream_max_mb", "target_expression": "asset['archive_stream_max_mb']", "value_expression": "options.get('pdmx_archive_stream_max_mb')" }, { "line": 851, "value_line": 851, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'archive_stream_required'" }, { "line": 853, "value_line": 853, "key": "download_status", "target_expression": "asset['download_status']", "value_expression": "'local_snapshot_missing'" } ] }, { "symbol": "DigitalSheetMusicCrawler._pdmx_row_matches", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 814, "line_end": 830, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 821, "receiver": "options", "key": "license_filter", "expression": "options.get('license_filter')" }, { "line": 824, "receiver": "options", "key": "query", "expression": "options.get('query')" }, { "line": 826, "receiver": "options", "key": "organ_only", "expression": "options.get('organ_only')" }, { "line": 827, "receiver": "self.config", "key": "pdmx_include_track_19", "expression": "self.config.get('pdmx_include_track_19', True)" }, { "line": 815, "receiver": "row", "key": "subset:deduplicated", "expression": "row.get('subset:deduplicated', '')" }, { "line": 817, "receiver": "row", "key": "subset:no_license_conflict", "expression": "row.get('subset:no_license_conflict', '')" }, { "line": 819, "receiver": "row", "key": "subset:all_valid", "expression": "row.get('subset:all_valid', '')" }, { "line": 821, "receiver": "row", "key": "license", "expression": "row.get('license')" }, { "line": 827, "receiver": "row", "key": "tracks", "expression": "row.get('tracks')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._raw_record", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 565, "line_end": 599, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 581, "receiver": "asset", "key": "format", "expression": "asset.get('format')" }, { "line": 581, "receiver": "asset", "key": "url", "expression": "asset.get('url')" }, { "line": 581, "receiver": "asset", "key": "local_path", "expression": "asset.get('local_path')" } ], "html_queries": [], "dictionary_entries": [ { "line": 583, "key": "score_id", "value_expression": "score_id" }, { "line": 584, "key": "source_collection", "value_expression": "source_collection" }, { "line": 585, "key": "title", "value_expression": "title" }, { "line": 586, "key": "composer", "value_expression": "composer" }, { "line": 587, "key": "source_url", "value_expression": "source_url" }, { "line": 588, "key": "url", "value_expression": "source_url" }, { "line": 589, "key": "license_name", "value_expression": "license_name" }, { "line": 590, "key": "license_url", "value_expression": "license_url" }, { "line": 591, "key": "catalog_number", "value_expression": "catalog_number" }, { "line": 592, "key": "instrumentation", "value_expression": "instrumentation" }, { "line": 593, "key": "formats", "value_expression": "','.join(formats)" }, { "line": 594, "key": "status", "value_expression": "status" }, { "line": 595, "key": "assets", "value_expression": "assets" }, { "line": 596, "key": "raw_data_json", "value_expression": "json.dumps({**raw_data, 'assets': assets}, ensure_ascii=False)" }, { "line": 597, "key": "raw_data", "value_expression": "raw_data" }, { "line": 598, "key": "scraped_at", "value_expression": "datetime.now().isoformat(timespec='seconds')" }, { "line": 596, "key": "assets", "value_expression": "assets" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler._respectful_pause", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 1021, "line_end": 1024, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 1022, "receiver": "self.config", "key": "rate_limit_delay", "expression": "self.config.get('rate_limit_delay', 1.5)" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 145, "line_end": 164, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_collection", "_crawl_options", "_persist_cache_records", "_respectful_pause", "_upsert_scrape_status", "is_scrape_stop_requested", "mark_scrape_stopped", "preflight_crawl" ], "literal_get_calls": [ { "line": 148, "receiver": "kwargs", "key": "task_id", "expression": "kwargs.get('task_id')" }, { "line": 150, "receiver": "preflight", "key": "ok", "expression": "preflight.get('ok')" }, { "line": 151, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" }, { "line": 152, "receiver": "preflight", "key": "message", "expression": "preflight.get('message')" } ], "html_queries": [], "dictionary_entries": [ { "line": 149, "key": "collection", "value_expression": "','.join(collections)" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.preflight_crawl", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 122, "line_end": 143, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [ "_crawl_options" ], "literal_get_calls": [], "html_queries": [], "dictionary_entries": [ { "line": 143, "key": "ok", "value_expression": "True" }, { "line": 125, "key": "ok", "value_expression": "False" }, { "line": 125, "key": "message", "value_expression": "'No supported digital sheet music collections were selected'" }, { "line": 140, "key": "ok", "value_expression": "False" }, { "line": 141, "key": "message", "value_expression": "'musescore_direct requires explicit URLs; an empty credential boundary is not a completed crawl'" }, { "line": 129, "key": "ok", "value_expression": "False" }, { "line": 129, "key": "message", "value_expression": "\"pdmx_mode must be either 'rows' or 'inventory'\"" }, { "line": 132, "key": "ok", "value_expression": "False" }, { "line": 134, "key": "message", "value_expression": "'PDMX row acquisition requires an existing pdmx_csv_path; use pdmx_mode=inventory only for an explicit dataset-inventory scope'" } ], "subscript_assignments": [] }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "line_start": 166, "line_end": 179, "inherited": false, "file_sha256": "4e8ac26682e90c7d14fce8f1d69edda9beb571f8c8bc7adce368d88183330694", "self_calls": [], "literal_get_calls": [ { "line": 167, "receiver": "raw_data", "key": "score_id", "expression": "raw_data.get('score_id')" }, { "line": 167, "receiver": "raw_data", "key": "id", "expression": "raw_data.get('id')" }, { "line": 169, "receiver": "payload", "key": "raw_data_json", "expression": "payload.get('raw_data_json')" }, { "line": 178, "receiver": "raw_data", "key": "transform_run_id", "expression": "raw_data.get('transform_run_id')" } ], "html_queries": [], "dictionary_entries": [], "subscript_assignments": [] } ], "selected_rule_rows": [ { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "score_id", "value_expression": "raw_data.get('score_id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "id", "value_expression": "raw_data.get('id')", "line": 167, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "raw_data_json", "value_expression": "payload.get('raw_data_json')", "line": 169, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.transform", "path": "aggregator/digital_sheet_music_crawler.py", "key": "transform_run_id", "value_expression": "raw_data.get('transform_run_id')", "line": 178, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "task_id", "value_expression": "kwargs.get('task_id')", "line": 148, "kind": "input read" }, { "symbol": "DigitalSheetMusicCrawler.crawl", "path": "aggregator/digital_sheet_music_crawler.py", "key": "ok", "value_expression": "preflight.get('ok')", "line": 150, "kind": "input read" } ], "bounded_code_passages": [ { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 172, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 166, "line_end": 174, "text": " def transform(self, raw_data: Dict[str, Any]) -> Dict[str, Any]:\n score_id = str(raw_data.get(\"score_id\") or raw_data.get(\"id\") or \"\")\n payload = dict(raw_data)\n if isinstance(payload.get(\"raw_data_json\"), str) and payload[\"raw_data_json\"]:\n try:\n payload.update(json.loads(payload[\"raw_data_json\"]))\n except json.JSONDecodeError:\n pass\n return normalize_digital_sheet_music_record(" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.transform", "line_start": 175, "line_end": 179, "text": " payload,\n primary_id=score_id,\n raw_content=json.dumps(payload, ensure_ascii=False, sort_keys=True),\n transform_run_id=raw_data.get(\"transform_run_id\"),\n )" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 145, "line_end": 153, "text": " def crawl(self, target: Any = None, **kwargs) -> List[Dict[str, Any]]:\n options = self._crawl_options(target, kwargs)\n collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))" }, { "path": "aggregator/digital_sheet_music_crawler.py", "symbol": "DigitalSheetMusicCrawler.crawl", "line_start": 147, "line_end": 155, "text": " collections = options[\"collections\"]\n task_id = kwargs.get(\"task_id\")\n preflight = self.preflight_crawl({**kwargs, \"collection\": \",\".join(collections), **options})\n if not preflight.get(\"ok\"):\n self._upsert_scrape_status(task_id, \"FAILED\", str(preflight.get(\"message\")), 0, len(collections))\n raise RuntimeError(str(preflight.get(\"message\")))\n self._upsert_scrape_status(task_id, \"RUNNING\", \"Starting digital sheet music crawl.\", 0, max(1, len(collections)))\n records: List[Dict[str, Any]] = []\n for index, collection in enumerate(collections, start=1):" } ] } }, { "catalogue_id": "S059", "source_key": "audit_orkasa", "name": "ORKASA", "selection_role": "audit_only", "parent_source_key": null, "attribution": "Evangelisch-Lutherische Landeskirche Sachsens", "scope_description": "Organs associated with the Saxon regional church", "documentary_unit": "Church organ database; no registered Aggregator adapter", "source_description": "The church's organ-information page identifies ORKASA as an organ database. Aggregator's dated investigation classified its access as requiring an external application client rather than ordinary HTTP acquisition.", "source_description_references": [ "audit_orkasa", "repo_audit" ], "interpretation": "That classification is a recorded implementation boundary, not a judgement of research value. ORKASA and the Dresden-led Saxony inventory have different institutional scopes and must remain separate entries.", "rights_observation": "No data-access or redistribution permission was established in this review.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "audit_orkasa", "repo_audit" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "bounded_audit_account", "acquisition": "No independently registered crawler is present for this audit-only entry. The cited investigation and public source account define the available evidence.", "transformation": "No source-specific transformation is attributed to this resource. Inferring a field model from the common MODAVIS schema would manufacture evidence.", "limitations": "That classification is a recorded implementation boundary, not a judgement of research value. ORKASA and the Dresden-led Saxony inventory have different institutional scopes and must remain separate entries." }, "implementation": null }, { "catalogue_id": "S060", "source_key": "audit_organstops", "name": "Encyclopedia of Organ Stops", "selection_role": "audit_only", "parent_source_key": null, "attribution": "Edward L. Stauff", "scope_description": "Encyclopedia of Organ Stops", "documentary_unit": "Organ-stop concept entry", "source_description": "The encyclopedia provides alphabetical access to stop terminology and names Edward L. Stauff as owner, operator and copyright holder. Aggregator's investigation excluded it from the organ-instance acquisition lane.", "source_description_references": [ "audit_organstops", "repo_audit" ], "interpretation": "That exclusion concerns record type. The encyclopedia remains relevant to terminology research, but a stop-name definition cannot identify a physical organ or prove that two historical stop names denote the same construction.", "rights_observation": "The site asserts copyright; no general open licence was established.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "audit_organstops", "repo_audit" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "bounded_audit_account", "acquisition": "No independently registered crawler is present for this audit-only entry. The cited investigation and public source account define the available evidence.", "transformation": "No source-specific transformation is attributed to this resource. Inferring a field model from the common MODAVIS schema would manufacture evidence.", "limitations": "That exclusion concerns record type. The encyclopedia remains relevant to terminology research, but a stop-name definition cannot identify a physical organ or prove that two historical stop names denote the same construction." }, "implementation": null }, { "catalogue_id": "S061", "source_key": "audit_welt_orgel_archiv", "name": "Welt-Orgel-Archiv", "selection_role": "audit_only", "parent_source_key": null, "attribution": "Welt-Orgel-Archiv (title recorded in the investigation audit)", "scope_description": "Not recoverable from the checked endpoint", "documentary_unit": "Unavailable historical web resource", "source_description": "The endpoint recorded in Aggregator's investigation returned HTTP 404 again on 10 September 2026. No content was recovered and no registered adapter is present.", "source_description_references": [ "audit_welt_orgel_archiv", "repo_audit" ], "interpretation": "Retain the URL and dated failure as evidence of the investigation's scope. The response does not establish the collection's historical extent, ownership, permanent disappearance or availability at another address.", "rights_observation": "Not assessable from the error response.", "rights_observation_basis": "Report of the pages consulted on 10 September 2026; not a determination of legal entitlement or a complete licence audit.", "references": [ "audit_welt_orgel_archiv", "repo_audit" ], "observation_date": "2026-09-10", "atlas_editorial_reading": { "evidence_kind": "bounded_audit_account", "acquisition": "No independently registered crawler is present for this audit-only entry. The cited investigation and public source account define the available evidence.", "transformation": "No source-specific transformation is attributed to this resource. Inferring a field model from the common MODAVIS schema would manufacture evidence.", "limitations": "Retain the URL and dated failure as evidence of the investigation's scope. The response does not establish the collection's historical extent, ownership, permanent disappearance or availability at another address." }, "implementation": null } ], "shared_helper_modules": [ { "path": "aggregator/transfer_schema.py", "sha256": "66e7c54c824cf418966ca7c8542e408217cf80130f1e447b7395f91e7b9d31d2", "functions": [ { "symbol": "canonical_source_key", "line_start": 287, "line_end": 295, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "empty_transfer_record", "line_start": 298, "line_end": 302, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "normalize_transfer_record", "line_start": 305, "line_end": 353, "literal_get_calls": [ { "line": 323, "key": "source_info", "expression": "payload.get('source_info')" }, { "line": 320, "key": "key", "expression": "payload.get('source', {}).get('key')" }, { "line": 320, "key": "source", "expression": "payload.get('source', {})" } ], "dictionary_entries": [] }, { "symbol": "_apply_source", "line_start": 356, "line_end": 367, "literal_get_calls": [ { "line": 359, "key": "key", "expression": "source.get('key', '')" }, { "line": 360, "key": "name", "expression": "metadata.get('name')" }, { "line": 360, "key": "name", "expression": "source.get('name')" }, { "line": 360, "key": "name", "expression": "(source_info or {}).get('name', '')" }, { "line": 361, "key": "base_url", "expression": "metadata.get('base_url')" }, { "line": 361, "key": "base_url", "expression": "source.get('base_url', '')" }, { "line": 364, "key": "retrieval_date", "expression": "source_info.get('retrieval_date')" }, { "line": 365, "key": "retrieved_at", "expression": "source_info.get('retrieved_at')" }, { "line": 366, "key": "retrieved_at", "expression": "source.get('retrieved_at', '')" } ], "dictionary_entries": [] }, { "symbol": "_merge_known_roots", "line_start": 370, "line_end": 386, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_normalize_source_snapshot", "line_start": 389, "line_end": 412, "literal_get_calls": [ { "line": 397, "key": "source_info", "expression": "payload.get('source_info')" }, { "line": 398, "key": "url", "expression": "snapshot.get('url')" }, { "line": 398, "key": "url", "expression": "source_info.get('url')" }, { "line": 398, "key": "url", "expression": "payload.get('url')" }, { "line": 398, "key": "link", "expression": "payload.get('link')" }, { "line": 402, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 402, "key": "html", "expression": "payload.get('html')" }, { "line": 404, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 410, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 412, "key": "documents", "expression": "snapshot.get('documents')" }, { "line": 397, "key": "source_info", "expression": "payload.get('source_info')" }, { "line": 400, "key": "raw_content", "expression": "snapshot.get('raw_content')" }, { "line": 400, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 400, "key": "html", "expression": "payload.get('html')" }, { "line": 403, "key": "content_type", "expression": "snapshot.get('content_type')" }, { "line": 404, "key": "content_type", "expression": "snapshot.get('content_type')" }, { "line": 405, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 409, "key": "content_sha256", "expression": "snapshot.get('content_sha256')" }, { "line": 411, "key": "html_sha256", "expression": "snapshot.get('html_sha256')" } ], "dictionary_entries": [] }, { "symbol": "_normalize_identifiers", "line_start": 415, "line_end": 467, "literal_get_calls": [ { "line": 419, "key": "primary_identifier", "expression": "SOURCE_METADATA.get(key, {}).get('primary_identifier')" }, { "line": 416, "key": "identifiers", "expression": "record.get('identifiers')" }, { "line": 417, "key": "other_identifiers", "expression": "record.get('other_identifiers')" }, { "line": 423, "key": "id", "expression": "payload.get('id')" }, { "line": 450, "key": "organ_identifiers", "expression": "payload.get('organ_identifiers')" }, { "line": 453, "key": "scheme", "expression": "item.get('scheme')" }, { "line": 453, "key": "name", "expression": "item.get('name')" }, { "line": 453, "key": "type", "expression": "item.get('type')" }, { "line": 454, "key": "value", "expression": "item.get('value')" }, { "line": 454, "key": "id", "expression": "item.get('id')" }, { "line": 461, "key": "value", "expression": "item.get('value')" }, { "line": 461, "key": "id", "expression": "item.get('id')" }, { "line": 462, "key": "scheme", "expression": "item.get('scheme')" }, { "line": 462, "key": "name", "expression": "item.get('name')" }, { "line": 462, "key": "hostcode", "expression": "item.get('hostcode')" } ], "dictionary_entries": [ { "line": 429, "key": "scheme", "value_expression": "source_primary" }, { "line": 430, "key": "value", "value_expression": "str(primary_id)" }, { "line": 431, "key": "role", "value_expression": "'primary'" }, { "line": 432, "key": "source", "value_expression": "key" }, { "line": 443, "key": "scheme", "value_expression": "legacy_name" }, { "line": 444, "key": "value", "value_expression": "str(legacy_value)" }, { "line": 445, "key": "role", "value_expression": "'source'" }, { "line": 446, "key": "source", "value_expression": "legacy_key" }, { "line": 456, "key": "scheme", "value_expression": "scheme" }, { "line": 456, "key": "value", "value_expression": "str(value)" }, { "line": 456, "key": "role", "value_expression": "'source'" }, { "line": 456, "key": "source", "value_expression": "key" }, { "line": 464, "key": "scheme", "value_expression": "scheme" }, { "line": 464, "key": "value", "value_expression": "str(value)" }, { "line": 464, "key": "role", "value_expression": "'alternate'" }, { "line": 464, "key": "source", "value_expression": "key" } ] }, { "symbol": "_normalize_locations", "line_start": 470, "line_end": 479, "literal_get_calls": [ { "line": 472, "key": "location", "expression": "payload.get('location')" }, { "line": 471, "key": "locations", "expression": "record.get('locations')" } ], "dictionary_entries": [] }, { "symbol": "_normalize_involved_and_events", "line_start": 482, "line_end": 548, "literal_get_calls": [ { "line": 483, "key": "involved", "expression": "record.get('involved')" }, { "line": 484, "key": "events", "expression": "record.get('events')" }, { "line": 509, "key": "builder_history", "expression": "payload.get('builder_history')" }, { "line": 526, "key": "description", "expression": "builder.get('description')" }, { "line": 539, "key": "key", "expression": "record.get('source', {}).get('key')" }, { "line": 489, "key": "dates", "expression": "event.get('dates')" }, { "line": 490, "key": "involved", "expression": "event.get('involved')" }, { "line": 493, "key": "name", "expression": "participant.get('name')" }, { "line": 512, "key": "builder_name", "expression": "builder.get('builder_name')" }, { "line": 512, "key": "name", "expression": "builder.get('name')" }, { "line": 513, "key": "year", "expression": "builder.get('year')" }, { "line": 513, "key": "date", "expression": "builder.get('date')" }, { "line": 514, "key": "activity", "expression": "builder.get('activity')" }, { "line": 496, "key": "activities", "expression": "participant.get('activities')" }, { "line": 497, "key": "type", "expression": "event.get('type')" }, { "line": 505, "key": "dates", "expression": "participant.get('dates')" }, { "line": 515, "key": "participants", "expression": "builder.get('participants')" }, { "line": 539, "key": "source", "expression": "record.get('source', {})" }, { "line": 498, "key": "type", "expression": "event.get('type')" }, { "line": 503, "key": "dates", "expression": "participant.get('dates')" }, { "line": 506, "key": "dates", "expression": "participant.get('dates')" } ], "dictionary_entries": [ { "line": 521, "key": "type", "value_expression": "activity" }, { "line": 522, "key": "dates", "value_expression": "[year] if year else []" }, { "line": 523, "key": "involved", "value_expression": "participants" }, { "line": 500, "key": "name", "value_expression": "name" }, { "line": 501, "key": "activities", "value_expression": "activities" }, { "line": 519, "key": "dates", "value_expression": "[year] if year else []" }, { "line": 516, "key": "name", "value_expression": "name" }, { "line": 516, "key": "activities", "value_expression": "[activity]" } ] }, { "symbol": "_deduplicate_orgbase_involved", "line_start": 551, "line_end": 581, "literal_get_calls": [ { "line": 571, "key": "source_occurrence_count", "expression": "current.get('source_occurrence_count')" }, { "line": 559, "key": "name", "expression": "value.get('name')" } ], "dictionary_entries": [] }, { "symbol": "_deduplicate_orgbase_events", "line_start": 584, "line_end": 625, "literal_get_calls": [ { "line": 623, "key": "derivation", "expression": "event.get('derivation')" }, { "line": 610, "key": "source_occurrence_count", "expression": "current.get('source_occurrence_count')" }, { "line": 618, "key": "involved", "expression": "current.get('involved')" }, { "line": 618, "key": "involved", "expression": "event.get('involved')" } ], "dictionary_entries": [] }, { "symbol": "_orgbase_event_key", "line_start": 628, "line_end": 649, "literal_get_calls": [ { "line": 630, "key": "type", "expression": "event.get('type')" }, { "line": 644, "key": "name", "expression": "item.get('name')" }, { "line": 641, "key": "dates", "expression": "event.get('dates')" }, { "line": 645, "key": "involved", "expression": "event.get('involved')" }, { "line": 630, "key": "activities", "expression": "event.get('activities')" }, { "line": 640, "key": "type", "expression": "event.get('type')" }, { "line": 646, "key": "name", "expression": "item.get('name')" } ], "dictionary_entries": [] }, { "symbol": "_semantic_text", "line_start": 652, "line_end": 653, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_orgbase_event_from_involved", "line_start": 656, "line_end": 694, "literal_get_calls": [ { "line": 661, "key": "dates", "expression": "participant.get('dates')" }, { "line": 662, "key": "activities", "expression": "participant.get('activities')" }, { "line": 686, "key": "entity_id", "expression": "participant.get('entity_id')" }, { "line": 659, "key": "name", "expression": "participant.get('name')" }, { "line": 660, "key": "raw_str", "expression": "participant.get('raw_str')" }, { "line": 684, "key": "activities", "expression": "participant.get('activities')" } ], "dictionary_entries": [ { "line": 679, "key": "type", "value_expression": "event_type" }, { "line": 680, "key": "activities", "value_expression": "[event_type]" }, { "line": 681, "key": "dates", "value_expression": "supported_dates" }, { "line": 682, "key": "involved", "value_expression": "[{'name': name, 'activities': list(participant.get('activities') or []), 'dates': supported_dates, 'entity_id': participant.get('entity_id')}]" }, { "line": 688, "key": "description", "value_expression": "raw_str" }, { "line": 689, "key": "raw_str", "value_expression": "raw_str" }, { "line": 690, "key": "derivation", "value_expression": "{'policy': 'orgbase-dated-builder-event:v1', 'evidence': 'raw_builder_statement_contains_date'}" }, { "line": 691, "key": "policy", "value_expression": "'orgbase-dated-builder-event:v1'" }, { "line": 692, "key": "evidence", "value_expression": "'raw_builder_statement_contains_date'" }, { "line": 683, "key": "name", "value_expression": "name" }, { "line": 684, "key": "activities", "value_expression": "list(participant.get('activities') or [])" }, { "line": 685, "key": "dates", "value_expression": "supported_dates" }, { "line": 686, "key": "entity_id", "value_expression": "participant.get('entity_id')" } ] }, { "symbol": "_normalize_specifications", "line_start": 697, "line_end": 724, "literal_get_calls": [ { "line": 702, "key": "stops", "expression": "specs.get('stops')" }, { "line": 703, "key": "stops_by_division", "expression": "specs.get('stops_by_division')" } ], "dictionary_entries": [ { "line": 720, "key": "raw_str", "value_expression": "str(stops)" } ] }, { "symbol": "_normalize_media", "line_start": 727, "line_end": 735, "literal_get_calls": [ { "line": 734, "key": "url", "expression": "image.get('url')" }, { "line": 734, "key": "link", "expression": "image.get('link')" } ], "dictionary_entries": [] }, { "symbol": "_normalize_literature", "line_start": 738, "line_end": 747, "literal_get_calls": [ { "line": 739, "key": "literature", "expression": "record.get('literature')" }, { "line": 740, "key": "sources", "expression": "payload.get('sources')" }, { "line": 742, "key": "citation", "expression": "source.get('citation')" }, { "line": 742, "key": "raw_str", "expression": "source.get('raw_str')" }, { "line": 742, "key": "title", "expression": "source.get('title')" } ], "dictionary_entries": [ { "line": 744, "key": "type", "value_expression": "'source_citation'" }, { "line": 744, "key": "raw_str", "value_expression": "citation" }, { "line": 746, "key": "type", "value_expression": "'source_citation'" }, { "line": 746, "key": "raw_str", "value_expression": "str(source)" } ] }, { "symbol": "_normalize_links", "line_start": 750, "line_end": 762, "literal_get_calls": [ { "line": 759, "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 752, "key": "links", "expression": "record.get('links')" }, { "line": 759, "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 754, "key": "url", "expression": "link.get('url')" }, { "line": 761, "key": "base_url", "expression": "record.get('source', {}).get('base_url', '')" }, { "line": 754, "key": "link", "expression": "link.get('link')" }, { "line": 761, "key": "source", "expression": "record.get('source', {})" } ], "dictionary_entries": [ { "line": 761, "key": "type", "value_expression": "'source'" }, { "line": 761, "key": "link", "value_expression": "snapshot_url" }, { "line": 761, "key": "host", "value_expression": "record.get('source', {}).get('base_url', '')" }, { "line": 758, "key": "link", "value_expression": "str(link)" } ] }, { "symbol": "_normalize_descriptions", "line_start": 765, "line_end": 766, "literal_get_calls": [ { "line": 766, "key": "descriptions", "expression": "record.get('descriptions')" } ], "dictionary_entries": [] }, { "symbol": "_apply_provenance_identity", "line_start": 769, "line_end": 792, "literal_get_calls": [ { "line": 784, "key": "crawl_run_id", "expression": "identity.get('crawl_run_id', '')" }, { "line": 785, "key": "transform_run_id", "expression": "identity.get('transform_run_id', '')" }, { "line": 786, "key": "artifact_ids", "expression": "identity.get('artifact_ids')" } ], "dictionary_entries": [] }, { "symbol": "_record_identity_parts", "line_start": 795, "line_end": 813, "literal_get_calls": [ { "line": 810, "key": "url", "expression": "record.get('source_snapshot', {}).get('url')" }, { "line": 796, "key": "primary_identifier", "expression": "SOURCE_METADATA.get(key, {}).get('primary_identifier')" }, { "line": 800, "key": "identifiers", "expression": "record.get('identifiers')" }, { "line": 806, "key": "identifiers", "expression": "record.get('identifiers')" }, { "line": 810, "key": "source_snapshot", "expression": "record.get('source_snapshot', {})" }, { "line": 803, "key": "role", "expression": "item.get('role')" }, { "line": 803, "key": "value", "expression": "item.get('value')" }, { "line": 807, "key": "value", "expression": "item.get('value')" }, { "line": 804, "key": "scheme", "expression": "item.get('scheme')" }, { "line": 804, "key": "value", "expression": "item.get('value')" }, { "line": 808, "key": "scheme", "expression": "item.get('scheme')" }, { "line": 808, "key": "value", "expression": "item.get('value')" } ], "dictionary_entries": [] }, { "symbol": "_apply_document_identity", "line_start": 816, "line_end": 856, "literal_get_calls": [ { "line": 819, "key": "url", "expression": "snapshot.get('url', '')" }, { "line": 820, "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at', '')" }, { "line": 821, "key": "crawl_run_id", "expression": "identity.get('crawl_run_id', '')" }, { "line": 823, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 838, "key": "raw_content", "expression": "snapshot.get('raw_content')" }, { "line": 839, "key": "content_type", "expression": "snapshot.get('content_type', '')" }, { "line": 818, "key": "documents", "expression": "snapshot.get('documents')" }, { "line": 856, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 820, "key": "source", "expression": "record.get('source', {})" }, { "line": 856, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 830, "key": "html_sha256", "expression": "snapshot.get('html_sha256')" }, { "line": 847, "key": "content_sha256", "expression": "snapshot.get('content_sha256')" } ], "dictionary_entries": [ { "line": 828, "key": "document_id", "value_expression": "provenance.document_id(key, primary_id, 'raw_html')" }, { "line": 829, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 830, "key": "content_sha256", "value_expression": "snapshot.get('html_sha256') or provenance.content_sha256(raw_html)" }, { "line": 831, "key": "content_type", "value_expression": "'text/html'" }, { "line": 832, "key": "url", "value_expression": "source_url" }, { "line": 833, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 834, "key": "crawl_run_id", "value_expression": "crawl_run_id" }, { "line": 845, "key": "document_id", "value_expression": "provenance.document_id(key, primary_id, document_kind)" }, { "line": 846, "key": "document_kind", "value_expression": "document_kind" }, { "line": 847, "key": "content_sha256", "value_expression": "snapshot.get('content_sha256') or provenance.content_sha256(raw_content)" }, { "line": 848, "key": "content_type", "value_expression": "content_type" }, { "line": 849, "key": "url", "value_expression": "source_url" }, { "line": 850, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 851, "key": "crawl_run_id", "value_expression": "crawl_run_id" } ] }, { "symbol": "_apply_entity_ids", "line_start": 859, "line_end": 876, "literal_get_calls": [ { "line": 860, "key": "key", "expression": "record.get('source', {}).get('key', '')" }, { "line": 867, "key": "media", "expression": "record.get('media', {})" }, { "line": 872, "key": "specifications", "expression": "record.get('specifications', {})" }, { "line": 865, "key": "events", "expression": "record.get('events')" }, { "line": 860, "key": "source", "expression": "record.get('source', {})" } ], "dictionary_entries": [] }, { "symbol": "_assign_list_entity_ids", "line_start": 879, "line_end": 884, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_assign_event_entity_ids", "line_start": 887, "line_end": 930, "literal_get_calls": [ { "line": 919, "key": "entity_id", "expression": "item.get('entity_id')" }, { "line": 900, "key": "id", "expression": "item.get('id')" }, { "line": 900, "key": "event_id", "expression": "item.get('event_id')" }, { "line": 921, "key": "legacy_entity_ids", "expression": "item.get('legacy_entity_ids')" } ], "dictionary_entries": [] }, { "symbol": "_attach_extensions", "line_start": 933, "line_end": 956, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_flatten_stops_by_division", "line_start": 959, "line_end": 969, "literal_get_calls": [], "dictionary_entries": [ { "line": 967, "key": "name", "value_expression": "str(stop)" }, { "line": 967, "key": "raw_str", "value_expression": "str(stop)" }, { "line": 967, "key": "division", "value_expression": "division" } ] }, { "symbol": "_as_list", "line_start": 972, "line_end": 983, "literal_get_calls": [], "dictionary_entries": [ { "line": 983, "key": "raw_str", "value_expression": "str(value)" } ] }, { "symbol": "_append_unique", "line_start": 986, "line_end": 989, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_signature", "line_start": 992, "line_end": 997, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_deep_merge", "line_start": 1000, "line_end": 1007, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/digital_sheet_music_transfer_schema.py", "sha256": "dd652091e15347c1973c7b71613d7db0c5658db9307c9bfaf87c4cf7ed7de04e", "functions": [ { "symbol": "normalize_digital_sheet_music_record", "line_start": 75, "line_end": 186, "literal_get_calls": [ { "line": 98, "key": "catalog_number", "expression": "payload.get('catalog_number')" }, { "line": 100, "key": "instrumentation", "expression": "payload.get('instrumentation')" }, { "line": 118, "key": "license_name", "expression": "payload.get('license_name')" }, { "line": 120, "key": "license_url", "expression": "payload.get('license_url')" }, { "line": 87, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 88, "key": "source_collection", "expression": "payload.get('source_collection')" }, { "line": 90, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 90, "key": "base_url", "expression": "collection_def.get('base_url')" }, { "line": 94, "key": "work", "expression": "payload.get('work')" }, { "line": 101, "key": "pdmx", "expression": "payload.get('pdmx')" }, { "line": 117, "key": "license", "expression": "payload.get('license')" }, { "line": 86, "key": "score_id", "expression": "payload.get('score_id')" }, { "line": 91, "key": "raw_content", "expression": "payload.get('raw_content')" }, { "line": 91, "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 95, "key": "title", "expression": "payload.get('title')" }, { "line": 96, "key": "title", "expression": "work.get('title')" }, { "line": 97, "key": "composer", "expression": "payload.get('composer')" }, { "line": 99, "key": "catalog_number", "expression": "payload.get('catalog_number')" }, { "line": 101, "key": "pdmx", "expression": "payload.get('pdmx')" }, { "line": 119, "key": "license_name", "expression": "payload.get('license_name')" }, { "line": 121, "key": "license_url", "expression": "payload.get('license_url')" }, { "line": 150, "key": "ambitus_global_midi", "expression": "payload.get('ambitus_global_midi')" }, { "line": 181, "key": "extensions", "expression": "payload.get('extensions')" }, { "line": 184, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 116, "key": "format", "expression": "asset.get('format')" }, { "line": 140, "key": "display_name", "expression": "collection_def.get('display_name')" }, { "line": 141, "key": "base_url", "expression": "collection_def.get('base_url')" }, { "line": 142, "key": "type", "expression": "collection_def.get('type')" }, { "line": 143, "key": "credential_required", "expression": "collection_def.get('credential_required')" }, { "line": 151, "key": "ambitus_by_part", "expression": "payload.get('ambitus_by_part')" }, { "line": 113, "key": "assets", "expression": "payload.get('assets')" }, { "line": 105, "key": "tracks", "expression": "pdmx_row.get('tracks')" } ], "dictionary_entries": [ { "line": 124, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 125, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 126, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 127, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 128, "key": "record_type", "value_expression": "'digital_sheet_music'" }, { "line": 129, "key": "source", "value_expression": "{**SOURCE_DEFINITION, 'retrieved_at': retrieved_at, 'crawler_version': 'digital_sheet_music_v1', 'run_id': crawl_run_id or ''}" }, { "line": 135, "key": "identifiers", "value_expression": "[{'scheme': 'score_id', 'value': primary_id, 'role': 'primary', 'source': SOURCE_KEY}]" }, { "line": 138, "key": "collection", "value_expression": "{'key': collection_key, 'display_name': collection_def.get('display_name') or collection_key, 'base_url': collection_def.get('base_url') or '', 'type': collection_def.get('type') or 'other', 'credential_required': bool(collection_def.get('credential_required'))}" }, { "line": 145, "key": "work", "value_expression": "work" }, { "line": 146, "key": "assets", "value_expression": "assets" }, { "line": 147, "key": "formats", "value_expression": "formats" }, { "line": 148, "key": "license", "value_expression": "license_payload" }, { "line": 149, "key": "analysis", "value_expression": "{'ambitus_global_midi': payload.get('ambitus_global_midi'), 'ambitus_by_part': payload.get('ambitus_by_part') or [], 'ambitus_projection_ready': False, 'notes': 'Ambitus extraction/projection intentionally deferred for this source-class implementation.'}" }, { "line": 155, "key": "source_snapshot", "value_expression": "_source_snapshot(primary_id=primary_id, source_url=source_url, raw_content=document_content, retrieved_at=retrieved_at, crawl_run_id=crawl_run_id)" }, { "line": 162, "key": "quality", "value_expression": "_quality_payload(payload, assets)" }, { "line": 163, "key": "provenance", "value_expression": "{'identity': {'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'score_id', 'primary_id_value': primary_id, 'crawl_run_id': crawl_run_id or '', 'transform_run_id': transform_run_id or '', 'artifact_ids': list(artifact_ids or [])}, 'schema': {'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}, 'normalizer': 'aggregator.digital_sheet_music_transfer_schema.normalize_digital_sheet_music_record'}" }, { "line": 181, "key": "extensions", "value_expression": "_clean_mapping(payload.get('extensions'))" }, { "line": 131, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 132, "key": "crawler_version", "value_expression": "'digital_sheet_music_v1'" }, { "line": 133, "key": "run_id", "value_expression": "crawl_run_id or ''" }, { "line": 139, "key": "key", "value_expression": "collection_key" }, { "line": 140, "key": "display_name", "value_expression": "collection_def.get('display_name') or collection_key" }, { "line": 141, "key": "base_url", "value_expression": "collection_def.get('base_url') or ''" }, { "line": 142, "key": "type", "value_expression": "collection_def.get('type') or 'other'" }, { "line": 143, "key": "credential_required", "value_expression": "bool(collection_def.get('credential_required'))" }, { "line": 150, "key": "ambitus_global_midi", "value_expression": "payload.get('ambitus_global_midi')" }, { "line": 151, "key": "ambitus_by_part", "value_expression": "payload.get('ambitus_by_part') or []" }, { "line": 152, "key": "ambitus_projection_ready", "value_expression": "False" }, { "line": 153, "key": "notes", "value_expression": "'Ambitus extraction/projection intentionally deferred for this source-class implementation.'" }, { "line": 164, "key": "identity", "value_expression": "{'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'score_id', 'primary_id_value': primary_id, 'crawl_run_id': crawl_run_id or '', 'transform_run_id': transform_run_id or '', 'artifact_ids': list(artifact_ids or [])}" }, { "line": 174, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 179, "key": "normalizer", "value_expression": "'aggregator.digital_sheet_music_transfer_schema.normalize_digital_sheet_music_record'" }, { "line": 136, "key": "scheme", "value_expression": "'score_id'" }, { "line": 136, "key": "value", "value_expression": "primary_id" }, { "line": 136, "key": "role", "value_expression": "'primary'" }, { "line": 136, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 165, "key": "source_id", "value_expression": "provenance.source_id(SOURCE_KEY)" }, { "line": 166, "key": "source_record_id", "value_expression": "source_record_id" }, { "line": 167, "key": "record_id", "value_expression": "source_record_id" }, { "line": 168, "key": "primary_id_scheme", "value_expression": "'score_id'" }, { "line": 169, "key": "primary_id_value", "value_expression": "primary_id" }, { "line": 170, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 171, "key": "transform_run_id", "value_expression": "transform_run_id or ''" }, { "line": 172, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 175, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 176, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 177, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" } ] }, { "symbol": "canonical_collection_key", "line_start": 189, "line_end": 191, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "normalize_title", "line_start": 194, "line_end": 195, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "slugify", "line_start": 198, "line_end": 202, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "normalize_url", "line_start": 205, "line_end": 212, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "asset_format_from_url", "line_start": 215, "line_end": 224, "literal_get_calls": [], "dictionary_entries": [ { "line": 224, "key": "extension", "value_expression": "ext" }, { "line": 224, "key": "format", "value_expression": "fmt" }, { "line": 224, "key": "content_type", "value_expression": "content_type" } ] }, { "symbol": "_normalize_asset", "line_start": 227, "line_end": 256, "literal_get_calls": [ { "line": 228, "key": "url", "expression": "asset.get('url')" }, { "line": 228, "key": "href", "expression": "asset.get('href')" }, { "line": 228, "key": "download_url", "expression": "asset.get('download_url')" }, { "line": 228, "key": "local_path", "expression": "asset.get('local_path')" }, { "line": 232, "key": "format", "expression": "asset.get('format')" }, { "line": 233, "key": "content_type", "expression": "asset.get('content_type')" }, { "line": 234, "key": "file_name", "expression": "asset.get('file_name')" }, { "line": 229, "key": "local_path", "expression": "asset.get('local_path')" }, { "line": 236, "key": "asset_id", "expression": "asset.get('asset_id')" }, { "line": 244, "key": "role", "expression": "asset.get('role')" }, { "line": 245, "key": "download_status", "expression": "asset.get('download_status')" }, { "line": 246, "key": "download_error", "expression": "asset.get('download_error')" }, { "line": 247, "key": "resolved_url", "expression": "asset.get('resolved_url')" }, { "line": 249, "key": "source_file_id", "expression": "asset.get('source_file_id')" }, { "line": 250, "key": "archive_url", "expression": "asset.get('archive_url')" }, { "line": 251, "key": "archive_member_path", "expression": "asset.get('archive_member_path')" }, { "line": 252, "key": "downloaded_path", "expression": "asset.get('downloaded_path')" }, { "line": 253, "key": "content_sha256", "expression": "asset.get('content_sha256')" }, { "line": 239, "key": "local_path", "expression": "asset.get('local_path')" }, { "line": 243, "key": "label", "expression": "asset.get('label')" }, { "line": 248, "key": "alternate_urls", "expression": "asset.get('alternate_urls')" }, { "line": 254, "key": "content_size_bytes", "expression": "asset.get('content_size_bytes')" }, { "line": 255, "key": "position", "expression": "asset.get('position')" } ], "dictionary_entries": [ { "line": 236, "key": "asset_id", "value_expression": "asset.get('asset_id') or provenance.entity_id(SOURCE_KEY, primary_id, 'asset', f'{index}:{url}')" }, { "line": 237, "key": "url", "value_expression": "'' if is_local else url" }, { "line": 238, "key": "normalized_url", "value_expression": "'' if is_local else normalize_url(url)" }, { "line": 239, "key": "local_path", "value_expression": "str(asset.get('local_path') or (url if is_local else ''))" }, { "line": 240, "key": "file_name", "value_expression": "file_name" }, { "line": 241, "key": "format", "value_expression": "fmt" }, { "line": 242, "key": "content_type", "value_expression": "content_type" }, { "line": 243, "key": "label", "value_expression": "normalize_title(asset.get('label') or fmt)" }, { "line": 244, "key": "role", "value_expression": "asset.get('role') or _asset_role(fmt)" }, { "line": 245, "key": "download_status", "value_expression": "asset.get('download_status') or 'discovered'" }, { "line": 246, "key": "download_error", "value_expression": "asset.get('download_error') or ''" }, { "line": 247, "key": "resolved_url", "value_expression": "asset.get('resolved_url') or ''" }, { "line": 248, "key": "alternate_urls", "value_expression": "list(asset.get('alternate_urls') or [])" }, { "line": 249, "key": "source_file_id", "value_expression": "asset.get('source_file_id') or ''" }, { "line": 250, "key": "archive_url", "value_expression": "asset.get('archive_url') or ''" }, { "line": 251, "key": "archive_member_path", "value_expression": "asset.get('archive_member_path') or ''" }, { "line": 252, "key": "downloaded_path", "value_expression": "asset.get('downloaded_path') or ''" }, { "line": 253, "key": "content_sha256", "value_expression": "asset.get('content_sha256') or ''" }, { "line": 254, "key": "content_size_bytes", "value_expression": "int(asset.get('content_size_bytes') or 0)" }, { "line": 255, "key": "position", "value_expression": "int(asset.get('position') or index)" } ] }, { "symbol": "_asset_role", "line_start": 259, "line_end": 268, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_source_snapshot", "line_start": 271, "line_end": 290, "literal_get_calls": [], "dictionary_entries": [ { "line": 274, "key": "document_id", "value_expression": "provenance.document_id(SOURCE_KEY, primary_id, 'raw_metadata')" }, { "line": 275, "key": "document_kind", "value_expression": "'raw_json'" }, { "line": 276, "key": "content_sha256", "value_expression": "content_hash" }, { "line": 277, "key": "content_size_bytes", "value_expression": "len(str(raw_content or '').encode('utf-8'))" }, { "line": 278, "key": "content_type", "value_expression": "'application/json'" }, { "line": 279, "key": "url", "value_expression": "source_url" }, { "line": 280, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 281, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 282, "key": "raw_content", "value_expression": "raw_content or ''" }, { "line": 285, "key": "url", "value_expression": "source_url" }, { "line": 286, "key": "content_type", "value_expression": "'application/json'" }, { "line": 287, "key": "content_sha256", "value_expression": "content_hash" }, { "line": 288, "key": "raw_content", "value_expression": "raw_content or ''" }, { "line": 289, "key": "documents", "value_expression": "[document]" } ] }, { "symbol": "_quality_payload", "line_start": 293, "line_end": 305, "literal_get_calls": [ { "line": 294, "key": "quality", "expression": "payload.get('quality')" }, { "line": 301, "key": "license_conflict", "expression": "payload.get('license_conflict')" }, { "line": 295, "key": "warnings", "expression": "quality.get('warnings')" }, { "line": 299, "key": "format", "expression": "asset.get('format')" } ], "dictionary_entries": [] }, { "symbol": "_with_entity_ids", "line_start": 308, "line_end": 316, "literal_get_calls": [ { "line": 313, "key": "assets", "expression": "record.get('assets')" } ], "dictionary_entries": [] }, { "symbol": "_clean_mapping", "line_start": 319, "line_end": 320, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/vmi_library_transfer_schema.py", "sha256": "752fb6c6e0a6991cfeb12115fb3b2a20ebe1e8d0be9f191f782004c36e971d0e", "functions": [ { "symbol": "normalize_vmi_library_record", "line_start": 140, "line_end": 228, "literal_get_calls": [ { "line": 159, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 161, "key": "catalog", "expression": "payload.get('catalog')" }, { "line": 162, "key": "item", "expression": "payload.get('item')" }, { "line": 163, "key": "package", "expression": "payload.get('package')" }, { "line": 164, "key": "classification", "expression": "payload.get('classification')" }, { "line": 165, "key": "organ_relation", "expression": "payload.get('organ_relation')" }, { "line": 173, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 173, "key": "catalog_url", "expression": "catalog.get('catalog_url')" }, { "line": 173, "key": "base_url", "expression": "source_def.get('base_url')" }, { "line": 184, "key": "quality", "expression": "payload.get('quality')" }, { "line": 160, "key": "item_id", "expression": "payload.get('item_id')" }, { "line": 175, "key": "raw_content", "expression": "payload.get('raw_content')" }, { "line": 175, "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 175, "key": "markdown", "expression": "payload.get('markdown')" }, { "line": 177, "key": "title", "expression": "payload.get('title')" }, { "line": 177, "key": "package_title", "expression": "package.get('package_title')" }, { "line": 178, "key": "title", "expression": "item.get('title')" }, { "line": 223, "key": "extensions", "expression": "payload.get('extensions')" }, { "line": 226, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 166, "key": "compatibility", "expression": "payload.get('compatibility')" }, { "line": 168, "key": "catalog_url", "expression": "catalog.get('catalog_url')" }, { "line": 168, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 168, "key": "base_url", "expression": "source_def.get('base_url')" }, { "line": 169, "key": "links", "expression": "payload.get('links')" }, { "line": 172, "key": "media", "expression": "payload.get('media')" }, { "line": 179, "key": "software_labels", "expression": "item.get('software_labels')" }, { "line": 180, "key": "engine_labels", "expression": "item.get('engine_labels')" } ], "dictionary_entries": [ { "line": 187, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 188, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 189, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 190, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 191, "key": "record_type", "value_expression": "record_type" }, { "line": 192, "key": "source", "value_expression": "_source_payload(source_key, source_def, retrieved_at, crawl_run_id)" }, { "line": 193, "key": "identifiers", "value_expression": "[{'scheme': id_scheme, 'value': primary_id, 'role': 'primary', 'source': source_key}]" }, { "line": 196, "key": "catalog", "value_expression": "catalog" }, { "line": 197, "key": "item", "value_expression": "item" }, { "line": 198, "key": "package", "value_expression": "package" }, { "line": 199, "key": "compatibility", "value_expression": "compatibility" }, { "line": 200, "key": "classification", "value_expression": "classification" }, { "line": 201, "key": "organ_relation", "value_expression": "organ_relation" }, { "line": 202, "key": "links", "value_expression": "links" }, { "line": 203, "key": "media", "value_expression": "media" }, { "line": 204, "key": "source_snapshot", "value_expression": "_source_snapshot(source_key=source_key, primary_id=primary_id, source_url=source_url, raw_content=document_content, document_kind=document_kind, retrieved_at=retrieved_at, crawl_run_id=crawl_run_id)" }, { "line": 213, "key": "quality", "value_expression": "quality" }, { "line": 214, "key": "provenance", "value_expression": "_provenance_payload(source_key=source_key, source_record_id=source_record_id, id_scheme=id_scheme, primary_id=primary_id, crawl_run_id=crawl_run_id, transform_run_id=transform_run_id, artifact_ids=artifact_ids)" }, { "line": 223, "key": "extensions", "value_expression": "_clean_mapping(payload.get('extensions'))" }, { "line": 194, "key": "scheme", "value_expression": "id_scheme" }, { "line": 194, "key": "value", "value_expression": "primary_id" }, { "line": 194, "key": "role", "value_expression": "'primary'" }, { "line": 194, "key": "source", "value_expression": "source_key" } ] }, { "symbol": "normalize_title", "line_start": 231, "line_end": 232, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "slugify", "line_start": 235, "line_end": 239, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "canonical_source_key", "line_start": 242, "line_end": 243, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_source_payload", "line_start": 246, "line_end": 256, "literal_get_calls": [ { "line": 249, "key": "name", "expression": "source_def.get('name')" }, { "line": 250, "key": "display_name", "expression": "source_def.get('display_name')" }, { "line": 251, "key": "base_url", "expression": "source_def.get('base_url')" }, { "line": 252, "key": "type", "expression": "source_def.get('type')" } ], "dictionary_entries": [ { "line": 248, "key": "key", "value_expression": "source_key" }, { "line": 249, "key": "name", "value_expression": "source_def.get('name') or source_key" }, { "line": 250, "key": "display_name", "value_expression": "source_def.get('display_name') or source_key" }, { "line": 251, "key": "base_url", "value_expression": "source_def.get('base_url') or ''" }, { "line": 252, "key": "type", "value_expression": "source_def.get('type') or 'web'" }, { "line": 253, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 254, "key": "crawler_version", "value_expression": "'grandorgue_library_v1'" }, { "line": 255, "key": "run_id", "value_expression": "crawl_run_id or ''" } ] }, { "symbol": "_source_snapshot", "line_start": 259, "line_end": 287, "literal_get_calls": [], "dictionary_entries": [ { "line": 272, "key": "document_id", "value_expression": "provenance.document_id(source_key, primary_id, document_kind)" }, { "line": 273, "key": "document_kind", "value_expression": "document_kind" }, { "line": 274, "key": "content_sha256", "value_expression": "content_hash" }, { "line": 275, "key": "content_size_bytes", "value_expression": "len(str(raw_content or '').encode('utf-8'))" }, { "line": 276, "key": "content_type", "value_expression": "content_type" }, { "line": 277, "key": "url", "value_expression": "source_url" }, { "line": 278, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 279, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 280, "key": "raw_content", "value_expression": "raw_content or ''" }, { "line": 283, "key": "url", "value_expression": "source_url" }, { "line": 284, "key": "content_type", "value_expression": "content_type" }, { "line": 285, "key": "content_sha256", "value_expression": "content_hash" }, { "line": 286, "key": "documents", "value_expression": "[document]" } ] }, { "symbol": "_provenance_payload", "line_start": 290, "line_end": 317, "literal_get_calls": [], "dictionary_entries": [ { "line": 301, "key": "identity", "value_expression": "{'source_id': provenance.source_id(source_key), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': id_scheme, 'primary_id_value': primary_id, 'crawl_run_id': crawl_run_id or '', 'transform_run_id': transform_run_id or '', 'artifact_ids': list(artifact_ids or [])}" }, { "line": 311, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 316, "key": "normalizer", "value_expression": "'aggregator.vmi_library_transfer_schema.normalize_vmi_library_record'" }, { "line": 302, "key": "source_id", "value_expression": "provenance.source_id(source_key)" }, { "line": 303, "key": "source_record_id", "value_expression": "source_record_id" }, { "line": 304, "key": "record_id", "value_expression": "source_record_id" }, { "line": 305, "key": "primary_id_scheme", "value_expression": "id_scheme" }, { "line": 306, "key": "primary_id_value", "value_expression": "primary_id" }, { "line": 307, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 308, "key": "transform_run_id", "value_expression": "transform_run_id or ''" }, { "line": 309, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 312, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 313, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 314, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" } ] }, { "symbol": "_normalize_link", "line_start": 320, "line_end": 364, "literal_get_calls": [ { "line": 323, "key": "link_text", "expression": "link.get('link_text')" }, { "line": 323, "key": "text", "expression": "link.get('text')" }, { "line": 323, "key": "label", "expression": "link.get('label')" }, { "line": 327, "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 328, "key": "is_odf_candidate", "expression": "link.get('is_odf_candidate')" }, { "line": 329, "key": "is_sample_package_candidate", "expression": "link.get('is_sample_package_candidate')" }, { "line": 330, "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 335, "key": "rel", "expression": "link.get('rel')" }, { "line": 336, "key": "content_type_hint", "expression": "link.get('content_type_hint')" }, { "line": 341, "key": "is_external_identifier_candidate", "expression": "link.get('is_external_identifier_candidate')" }, { "line": 343, "key": "source_selector", "expression": "link.get('source_selector')" }, { "line": 321, "key": "url", "expression": "link.get('url')" }, { "line": 321, "key": "href", "expression": "link.get('href')" }, { "line": 342, "key": "position", "expression": "link.get('position')" } ], "dictionary_entries": [ { "line": 332, "key": "url", "value_expression": "url" }, { "line": 333, "key": "normalized_url", "value_expression": "normalized_url" }, { "line": 334, "key": "link_text", "value_expression": "link_text" }, { "line": 335, "key": "rel", "value_expression": "link.get('rel') or _link_rel(lower, is_download)" }, { "line": 336, "key": "content_type_hint", "value_expression": "link.get('content_type_hint') or _content_type_hint(ext)" }, { "line": 337, "key": "is_download_candidate", "value_expression": "is_download" }, { "line": 338, "key": "is_product_homepage_candidate", "value_expression": "is_product_homepage" }, { "line": 339, "key": "is_odf_candidate", "value_expression": "is_odf" }, { "line": 340, "key": "is_sample_package_candidate", "value_expression": "is_sample" }, { "line": 341, "key": "is_external_identifier_candidate", "value_expression": "bool(link.get('is_external_identifier_candidate'))" }, { "line": 342, "key": "position", "value_expression": "int(link.get('position') or index)" }, { "line": 343, "key": "source_selector", "value_expression": "link.get('source_selector') or {}" }, { "line": 347, "key": "eligible", "value_expression": "True" }, { "line": 348, "key": "status", "value_expression": "'candidate'" }, { "line": 349, "key": "task", "value_expression": "'vmi_product_page_analysis'" }, { "line": 350, "key": "target_entity_path", "value_expression": "'item'" }, { "line": 351, "key": "expected_outputs", "value_expression": "['developer_or_publisher', 'represented_organ', 'source_institution', 'engine_or_platform', 'version', 'license_or_access_hints', 'package_or_download_metadata']" }, { "line": 360, "key": "requires_review", "value_expression": "True" }, { "line": 361, "key": "fetch_policy", "value_expression": "'public_html_only'" }, { "line": 362, "key": "reason", "value_expression": "'VMI catalog link points to a product or detail page where richer instrument evidence may exist.'" } ] }, { "symbol": "normalize_url", "line_start": 367, "line_end": 374, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_link_rel", "line_start": 377, "line_end": 384, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_content_type_hint", "line_start": 387, "line_end": 395, "literal_get_calls": [], "dictionary_entries": [ { "line": 389, "key": ".zip", "value_expression": "'application/zip'" }, { "line": 390, "key": ".rar", "value_expression": "'application/vnd.rar'" }, { "line": 391, "key": ".7z", "value_expression": "'application/x-7z-compressed'" }, { "line": 392, "key": ".pdf", "value_expression": "'application/pdf'" }, { "line": 393, "key": ".orgue", "value_expression": "'text/plain'" }, { "line": 394, "key": ".organ", "value_expression": "'text/plain'" } ] }, { "symbol": "_normalize_media", "line_start": 398, "line_end": 406, "literal_get_calls": [ { "line": 400, "key": "kind", "expression": "item.get('kind')" }, { "line": 401, "key": "url", "expression": "item.get('url')" }, { "line": 401, "key": "link", "expression": "item.get('link')" }, { "line": 403, "key": "file_name", "expression": "item.get('file_name')" }, { "line": 405, "key": "status", "expression": "item.get('status')" }, { "line": 402, "key": "link_text", "expression": "item.get('link_text')" }, { "line": 402, "key": "alt", "expression": "item.get('alt')" }, { "line": 404, "key": "position", "expression": "item.get('position')" }, { "line": 403, "key": "url", "expression": "item.get('url')" } ], "dictionary_entries": [ { "line": 400, "key": "kind", "value_expression": "item.get('kind') or 'media'" }, { "line": 401, "key": "url", "value_expression": "item.get('url') or item.get('link') or ''" }, { "line": 402, "key": "link_text", "value_expression": "normalize_title(item.get('link_text') or item.get('alt') or '')" }, { "line": 403, "key": "file_name", "value_expression": "item.get('file_name') or os.path.basename(urlparse(str(item.get('url') or '')).path)" }, { "line": 404, "key": "position", "value_expression": "int(item.get('position') or index)" }, { "line": 405, "key": "status", "value_expression": "item.get('status') or 'discovered'" } ] }, { "symbol": "_normalize_package_flags", "line_start": 409, "line_end": 424, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_quality_payload", "line_start": 427, "line_end": 434, "literal_get_calls": [ { "line": 432, "key": "title", "expression": "item.get('title')" }, { "line": 430, "key": "download_url", "expression": "package.get('download_url')" } ], "dictionary_entries": [] }, { "symbol": "_with_entity_ids", "line_start": 437, "line_end": 446, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_mapping", "line_start": 449, "line_end": 450, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_unique_labels", "line_start": 453, "line_end": 461, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/tuning_system_transfer_schema.py", "sha256": "29cb7c0b01f7d2311be94ca91ff48e78b7df2b1375226bf4c1cd180655a9ad42", "functions": [ { "symbol": "normalize_tuning_system_record", "line_start": 33, "line_end": 207, "literal_get_calls": [ { "line": 51, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 51, "key": "url", "expression": "payload.get('url')" }, { "line": 52, "key": "scraped_at", "expression": "payload.get('scraped_at')" }, { "line": 52, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 57, "key": "is_precise_variant", "expression": "payload.get('is_precise_variant')" }, { "line": 60, "key": "raw_commentary_html", "expression": "payload.get('raw_commentary_html')" }, { "line": 60, "key": "commentary_raw_html", "expression": "payload.get('commentary_raw_html')" }, { "line": 43, "key": "label", "expression": "payload.get('label')" }, { "line": 43, "key": "tuning_id", "expression": "payload.get('tuning_id')" }, { "line": 43, "key": "id", "expression": "payload.get('id')" }, { "line": 44, "key": "title", "expression": "payload.get('title')" }, { "line": 45, "key": "tuning_id", "expression": "payload.get('tuning_id')" }, { "line": 49, "key": "cent_values", "expression": "payload.get('cent_values')" }, { "line": 49, "key": "centstr", "expression": "payload.get('centstr')" }, { "line": 53, "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 53, "key": "raw_content", "expression": "payload.get('raw_content')" }, { "line": 76, "key": "group_key", "expression": "payload.get('group_key')" }, { "line": 77, "key": "group_title", "expression": "payload.get('group_title')" }, { "line": 102, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 144, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 202, "key": "source_authors", "expression": "payload.get('source_authors')" }, { "line": 105, "key": "page_key", "expression": "payload.get('page_key')" }, { "line": 114, "key": "url", "expression": "commentary.get('url')" }, { "line": 119, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 172, "key": "available", "expression": "commentary.get('available')" }, { "line": 173, "key": "description", "expression": "commentary.get('description')" }, { "line": 184, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 185, "key": "transform_run_id", "expression": "payload.get('transform_run_id')" } ], "dictionary_entries": [ { "line": 70, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, tuning_id, 'tuning_system', 0)" }, { "line": 71, "key": "tuning_id", "value_expression": "tuning_id" }, { "line": 72, "key": "label", "value_expression": "label" }, { "line": 73, "key": "title", "value_expression": "title" }, { "line": 74, "key": "source_url", "value_expression": "source_url" }, { "line": 75, "key": "source_page_is_provenance", "value_expression": "True" }, { "line": 76, "key": "group_key", "value_expression": "_clean_text(payload.get('group_key'))" }, { "line": 77, "key": "group_title", "value_expression": "_clean_text(payload.get('group_title'))" }, { "line": 78, "key": "group_memberships", "value_expression": "group_memberships" }, { "line": 79, "key": "is_precise_variant", "value_expression": "precise_variant" }, { "line": 80, "key": "tone_order", "value_expression": "TONE_ORDER" }, { "line": 81, "key": "cent_values", "value_expression": "cent_values" }, { "line": 82, "key": "centstr", "value_expression": "centstr" }, { "line": 83, "key": "cent_reference", "value_expression": "{'description': 'BDO centstr vector in source tone order; values are preserved before derived intervals.', 'source_parameter': 'centstr', 'unit': 'cent'}" }, { "line": 88, "key": "derived_intervals", "value_expression": "_derived_intervals(cent_values, precise_variant=precise_variant)" }, { "line": 135, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 136, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 137, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 138, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 139, "key": "record_type", "value_expression": "'tuning_system'" }, { "line": 140, "key": "source", "value_expression": "{**SOURCE_DEFINITION, 'retrieved_at': retrieved_at, 'crawler_version': 'bdo_tuning_systems_v1', 'run_id': crawl_run_id or payload.get('crawl_run_id') or ''}" }, { "line": 146, "key": "identifiers", "value_expression": "[{'scheme': 'bdo_label', 'value': tuning_id, 'role': 'primary', 'source': SOURCE_KEY}, {'scheme': 'source_label', 'value': label, 'role': 'local', 'source': SOURCE_KEY}]" }, { "line": 150, "key": "tuning_system", "value_expression": "tuning_system" }, { "line": 151, "key": "tuning_systems", "value_expression": "[tuning_system]" }, { "line": 152, "key": "commentary", "value_expression": "commentary" }, { "line": 153, "key": "literature", "value_expression": "literature" }, { "line": 154, "key": "links", "value_expression": "links" }, { "line": 155, "key": "source_snapshot", "value_expression": "{'url': source_url, 'content_type': 'text/html', 'raw_content': raw_document, 'raw_html': raw_document if '<' in str(raw_document) else '', 'raw_commentary_html': raw_commentary_html, 'content_sha256': provenance.content_sha256(raw_document or _identity_text(payload)), 'html_sha256': provenance.content_sha256(raw_document) if '<' in str(raw_document) else '', 'documents': documents}" }, { "line": 165, "key": "quality", "value_expression": "{'schema_valid': None, 'completeness': {'has_label': bool(label), 'has_title': bool(title), 'has_cent_values': len(cent_values) == len(TONE_ORDER), 'has_group': bool(tuning_system['group_key']), 'has_commentary': bool(commentary.get('available')), 'has_description': bool(commentary.get('description')), 'has_literature': bool(literature)}}" }, { "line": 177, "key": "provenance", "value_expression": "{'identity': {'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'bdo_label', 'primary_id_value': tuning_id, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [item['document_id'] for item in documents]}, 'schema': {'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}, 'normalizer': 'aggregator.tuning_system_transfer_schema.normalize_tuning_system_record'}" }, { "line": 196, "key": "extensions", "value_expression": "{'source_query': {'label': label, 'title': title, 'centstr': centstr}, 'source_authors': payload.get('source_authors') or ['Hans Eugen Frischknecht', 'Jakob Schmid'], 'source_license_note': 'Publicly fetchable source page; reuse rights not inferred by Aggregator.', 'raw_group_memberships': group_memberships}" }, { "line": 84, "key": "description", "value_expression": "'BDO centstr vector in source tone order; values are preserved before derived intervals.'" }, { "line": 85, "key": "source_parameter", "value_expression": "'centstr'" }, { "line": 86, "key": "unit", "value_expression": "'cent'" }, { "line": 95, "key": "document_id", "value_expression": "document_id" }, { "line": 96, "key": "document_kind", "value_expression": "'tuning_html'" }, { "line": 97, "key": "url", "value_expression": "source_url" }, { "line": 98, "key": "content_type", "value_expression": "'text/html'" }, { "line": 99, "key": "content_sha256", "value_expression": "document_hash" }, { "line": 100, "key": "content_size_bytes", "value_expression": "len(str(raw_document).encode('utf-8', errors='ignore'))" }, { "line": 101, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 102, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 103, "key": "metadata", "value_expression": "{'source_parameter_keys': ['label', 'title', 'centstr'], 'source_page_key': payload.get('page_key') or ''}" }, { "line": 142, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 143, "key": "crawler_version", "value_expression": "'bdo_tuning_systems_v1'" }, { "line": 144, "key": "run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 156, "key": "url", "value_expression": "source_url" }, { "line": 157, "key": "content_type", "value_expression": "'text/html'" }, { "line": 158, "key": "raw_content", "value_expression": "raw_document" }, { "line": 159, "key": "raw_html", "value_expression": "raw_document if '<' in str(raw_document) else ''" }, { "line": 160, "key": "raw_commentary_html", "value_expression": "raw_commentary_html" }, { "line": 161, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document or _identity_text(payload))" }, { "line": 162, "key": "html_sha256", "value_expression": "provenance.content_sha256(raw_document) if '<' in str(raw_document) else ''" }, { "line": 163, "key": "documents", "value_expression": "documents" }, { "line": 166, "key": "schema_valid", "value_expression": "None" }, { "line": 167, "key": "completeness", "value_expression": "{'has_label': bool(label), 'has_title': bool(title), 'has_cent_values': len(cent_values) == len(TONE_ORDER), 'has_group': bool(tuning_system['group_key']), 'has_commentary': bool(commentary.get('available')), 'has_description': bool(commentary.get('description')), 'has_literature': bool(literature)}" }, { "line": 178, "key": "identity", "value_expression": "{'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'bdo_label', 'primary_id_value': tuning_id, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [item['document_id'] for item in documents]}" }, { "line": 189, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 194, "key": "normalizer", "value_expression": "'aggregator.tuning_system_transfer_schema.normalize_tuning_system_record'" }, { "line": 197, "key": "source_query", "value_expression": "{'label': label, 'title': title, 'centstr': centstr}" }, { "line": 202, "key": "source_authors", "value_expression": "payload.get('source_authors') or ['Hans Eugen Frischknecht', 'Jakob Schmid']" }, { "line": 203, "key": "source_license_note", "value_expression": "'Publicly fetchable source page; reuse rights not inferred by Aggregator.'" }, { "line": 204, "key": "raw_group_memberships", "value_expression": "group_memberships" }, { "line": 104, "key": "source_parameter_keys", "value_expression": "['label', 'title', 'centstr']" }, { "line": 105, "key": "source_page_key", "value_expression": "payload.get('page_key') or ''" }, { "line": 112, "key": "document_id", "value_expression": "commentary_document_id" }, { "line": 113, "key": "document_kind", "value_expression": "'commentary_html'" }, { "line": 114, "key": "url", "value_expression": "commentary.get('url') or ''" }, { "line": 115, "key": "content_type", "value_expression": "'text/html'" }, { "line": 116, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_commentary_html)" }, { "line": 117, "key": "content_size_bytes", "value_expression": "len(str(raw_commentary_html).encode('utf-8', errors='ignore'))" }, { "line": 118, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 119, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 120, "key": "metadata", "value_expression": "{'source_page_key': 'commentary', 'source_label': label, 'source_sections': [key for key in ('description', 'author_note', 'source_note', 'literature_text') if commentary.get(key)]}" }, { "line": 147, "key": "scheme", "value_expression": "'bdo_label'" }, { "line": 147, "key": "value", "value_expression": "tuning_id" }, { "line": 147, "key": "role", "value_expression": "'primary'" }, { "line": 147, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 148, "key": "scheme", "value_expression": "'source_label'" }, { "line": 148, "key": "value", "value_expression": "label" }, { "line": 148, "key": "role", "value_expression": "'local'" }, { "line": 148, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 168, "key": "has_label", "value_expression": "bool(label)" }, { "line": 169, "key": "has_title", "value_expression": "bool(title)" }, { "line": 170, "key": "has_cent_values", "value_expression": "len(cent_values) == len(TONE_ORDER)" }, { "line": 171, "key": "has_group", "value_expression": "bool(tuning_system['group_key'])" }, { "line": 172, "key": "has_commentary", "value_expression": "bool(commentary.get('available'))" }, { "line": 173, "key": "has_description", "value_expression": "bool(commentary.get('description'))" }, { "line": 174, "key": "has_literature", "value_expression": "bool(literature)" }, { "line": 179, "key": "source_id", "value_expression": "provenance.source_id(SOURCE_KEY)" }, { "line": 180, "key": "source_record_id", "value_expression": "source_record_id" }, { "line": 181, "key": "record_id", "value_expression": "source_record_id" }, { "line": 182, "key": "primary_id_scheme", "value_expression": "'bdo_label'" }, { "line": 183, "key": "primary_id_value", "value_expression": "tuning_id" }, { "line": 184, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 185, "key": "transform_run_id", "value_expression": "transform_run_id or payload.get('transform_run_id') or ''" }, { "line": 186, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 187, "key": "document_ids", "value_expression": "[item['document_id'] for item in documents]" }, { "line": 190, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 191, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 192, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 198, "key": "label", "value_expression": "label" }, { "line": 199, "key": "title", "value_expression": "title" }, { "line": 200, "key": "centstr", "value_expression": "centstr" }, { "line": 121, "key": "source_page_key", "value_expression": "'commentary'" }, { "line": 122, "key": "source_label", "value_expression": "label" }, { "line": 123, "key": "source_sections", "value_expression": "[key for key in ('description', 'author_note', 'source_note', 'literature_text') if commentary.get(key)]" } ] }, { "symbol": "_derived_intervals", "line_start": 210, "line_end": 244, "literal_get_calls": [], "dictionary_entries": [ { "line": 232, "key": "calculation_basis", "value_expression": "'BDO init_svg.js table calculation'" }, { "line": 233, "key": "precision_variant", "value_expression": "'precise' if precise_variant else 'rounded'" }, { "line": 234, "key": "absolute_cents", "value_expression": "[{'tone': tone, 'cents': round(values[index] + OCTAVE_STEPS[index], precision)} for index, tone in enumerate(TONE_ORDER)]" }, { "line": 238, "key": "major_thirds", "value_expression": "thirds" }, { "line": 239, "key": "fifths", "value_expression": "fifths" }, { "line": 240, "key": "relative_to_a", "value_expression": "[{'tone': tone, 'cents': round(values[index] - values[3], precision)} for index, tone in enumerate(TONE_ORDER)]" }, { "line": 228, "key": "from", "value_expression": "tone" }, { "line": 228, "key": "to", "value_expression": "TONE_ORDER[third_index]" }, { "line": 228, "key": "cents", "value_expression": "round(values[third_index] - values[index] + third_correction, precision)" }, { "line": 229, "key": "from", "value_expression": "tone" }, { "line": 229, "key": "to", "value_expression": "TONE_ORDER[fifth_index]" }, { "line": 229, "key": "cents", "value_expression": "round(values[fifth_index] - values[index] + fifth_correction, precision)" }, { "line": 235, "key": "tone", "value_expression": "tone" }, { "line": 235, "key": "cents", "value_expression": "round(values[index] + OCTAVE_STEPS[index], precision)" }, { "line": 241, "key": "tone", "value_expression": "tone" }, { "line": 241, "key": "cents", "value_expression": "round(values[index] - values[3], precision)" } ] }, { "symbol": "_cent_values", "line_start": 247, "line_end": 271, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_group_memberships", "line_start": 274, "line_end": 295, "literal_get_calls": [ { "line": 275, "key": "group_memberships", "expression": "payload.get('group_memberships')" }, { "line": 285, "key": "group_key", "expression": "payload.get('group_key')" }, { "line": 285, "key": "group_title", "expression": "payload.get('group_title')" }, { "line": 290, "key": "group_key", "expression": "item.get('group_key')" }, { "line": 290, "key": "group_title", "expression": "item.get('group_title')" }, { "line": 281, "key": "group_key", "expression": "item.get('group_key')" }, { "line": 282, "key": "group_title", "expression": "item.get('group_title')" }, { "line": 286, "key": "group_key", "expression": "payload.get('group_key')" }, { "line": 286, "key": "group_title", "expression": "payload.get('group_title')" } ], "dictionary_entries": [ { "line": 286, "key": "group_key", "value_expression": "_clean_text(payload.get('group_key'))" }, { "line": 286, "key": "group_title", "value_expression": "_clean_text(payload.get('group_title'))" }, { "line": 284, "key": "group_key", "value_expression": "key" }, { "line": 284, "key": "group_title", "value_expression": "title" } ] }, { "symbol": "_commentary", "line_start": 298, "line_end": 328, "literal_get_calls": [ { "line": 299, "key": "commentary", "expression": "payload.get('commentary')" }, { "line": 300, "key": "literature_entries", "expression": "raw.get('literature_entries')" }, { "line": 300, "key": "literature_entries", "expression": "payload.get('literature_entries')" }, { "line": 304, "key": "literature_text", "expression": "raw.get('literature_text')" }, { "line": 304, "key": "literature_text", "expression": "payload.get('literature_text')" }, { "line": 299, "key": "commentary", "expression": "payload.get('commentary')" }, { "line": 306, "key": "available", "expression": "raw.get('available')" }, { "line": 306, "key": "commentary_available", "expression": "payload.get('commentary_available')" }, { "line": 318, "key": "url", "expression": "raw.get('url')" }, { "line": 318, "key": "commentary_url", "expression": "payload.get('commentary_url')" }, { "line": 319, "key": "description", "expression": "raw.get('description')" }, { "line": 319, "key": "description", "expression": "payload.get('description')" }, { "line": 320, "key": "author_note", "expression": "raw.get('author_note')" }, { "line": 320, "key": "author_note", "expression": "payload.get('author_note')" }, { "line": 321, "key": "source_note", "expression": "raw.get('source_note')" }, { "line": 321, "key": "source_note", "expression": "payload.get('source_note')" }, { "line": 324, "key": "raw_text", "expression": "raw.get('raw_text')" }, { "line": 324, "key": "commentary_raw_text", "expression": "payload.get('commentary_raw_text')" }, { "line": 309, "key": "description", "expression": "payload.get('description')" }, { "line": 310, "key": "author_note", "expression": "payload.get('author_note')" }, { "line": 311, "key": "source_note", "expression": "payload.get('source_note')" }, { "line": 312, "key": "literature_text", "expression": "payload.get('literature_text')" }, { "line": 313, "key": "raw_commentary_html", "expression": "payload.get('raw_commentary_html')" } ], "dictionary_entries": [ { "line": 317, "key": "available", "value_expression": "available" }, { "line": 318, "key": "url", "value_expression": "_clean_text(raw.get('url') or payload.get('commentary_url'))" }, { "line": 319, "key": "description", "value_expression": "_clean_text(raw.get('description') or payload.get('description'))" }, { "line": 320, "key": "author_note", "value_expression": "_clean_text(raw.get('author_note') or payload.get('author_note'))" }, { "line": 321, "key": "source_note", "value_expression": "_clean_text(raw.get('source_note') or payload.get('source_note'))" }, { "line": 322, "key": "literature_text", "value_expression": "str(literature_text or '').strip()" }, { "line": 323, "key": "literature_entries", "value_expression": "entries" }, { "line": 324, "key": "raw_text", "value_expression": "_clean_text(raw.get('raw_text') or payload.get('commentary_raw_text'))" } ] }, { "symbol": "_literature_records", "line_start": 331, "line_end": 351, "literal_get_calls": [ { "line": 334, "key": "literature_entries", "expression": "commentary.get('literature_entries')" }, { "line": 335, "key": "literature_text", "expression": "commentary.get('literature_text')" }, { "line": 336, "key": "literature_text", "expression": "commentary.get('literature_text')" }, { "line": 346, "key": "url", "expression": "commentary.get('url')" } ], "dictionary_entries": [ { "line": 344, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, tuning_id, 'literature', index)" }, { "line": 345, "key": "citation_text", "value_expression": "citation_text" }, { "line": 346, "key": "source_url", "value_expression": "commentary.get('url') or ''" }, { "line": 347, "key": "source_page_is_provenance", "value_expression": "True" }, { "line": 348, "key": "source_path", "value_expression": "f'commentary.literature_entries[{index}]'" } ] }, { "symbol": "_link_records", "line_start": 354, "line_end": 374, "literal_get_calls": [ { "line": 364, "key": "url", "expression": "commentary.get('url')" }, { "line": 368, "key": "url", "expression": "commentary.get('url')" } ], "dictionary_entries": [ { "line": 357, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, tuning_id, 'link', 0)" }, { "line": 358, "key": "url", "value_expression": "source_url" }, { "line": 359, "key": "label", "value_expression": "'BDO tuning visualization'" }, { "line": 360, "key": "kind", "value_expression": "'source_record'" }, { "line": 361, "key": "source_path", "value_expression": "'tuning_system.source_url'" }, { "line": 367, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, tuning_id, 'link', 1)" }, { "line": 368, "key": "url", "value_expression": "commentary.get('url')" }, { "line": 369, "key": "label", "value_expression": "'BDO tuning commentary'" }, { "line": 370, "key": "kind", "value_expression": "'commentary'" }, { "line": 371, "key": "source_path", "value_expression": "'commentary.url'" } ] }, { "symbol": "_template_url", "line_start": 377, "line_end": 378, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_identity_text", "line_start": 381, "line_end": 386, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_format_cent", "line_start": 389, "line_end": 392, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_identifier", "line_start": 395, "line_end": 400, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_text", "line_start": 403, "line_end": 404, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_read_bool", "line_start": 407, "line_end": 417, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/gdo_literature_transfer_schema.py", "sha256": "181dce7bff83d952eb4036fa00962f53e61be4f31c5148d79f60c25252027b10", "functions": [ { "symbol": "normalize_gdo_literature_record", "line_start": 28, "line_end": 146, "literal_get_calls": [ { "line": 40, "key": "scraped_at", "expression": "payload.get('scraped_at')" }, { "line": 40, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 41, "key": "url", "expression": "payload.get('url')" }, { "line": 41, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 49, "key": "query", "expression": "payload.get('query')" }, { "line": 38, "key": "content_sha256", "expression": "payload.get('content_sha256')" }, { "line": 39, "key": "literature_id", "expression": "payload.get('literature_id')" }, { "line": 42, "key": "citation_text", "expression": "payload.get('citation_text')" }, { "line": 42, "key": "full_citation", "expression": "payload.get('full_citation')" }, { "line": 43, "key": "author", "expression": "payload.get('author')" }, { "line": 45, "key": "title", "expression": "payload.get('title')" }, { "line": 46, "key": "notes", "expression": "payload.get('notes')" }, { "line": 46, "key": "annotation", "expression": "payload.get('annotation')" }, { "line": 48, "key": "publication_details", "expression": "payload.get('publication_details')" }, { "line": 65, "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 141, "key": "result_index", "expression": "payload.get('result_index')" }, { "line": 142, "key": "declared_total_records", "expression": "payload.get('declared_total_records')" }, { "line": 143, "key": "year_result_count", "expression": "payload.get('year_result_count')" }, { "line": 79, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 127, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 128, "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 103, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" } ], "dictionary_entries": [ { "line": 52, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, content_hash, 'literature', 0)" }, { "line": 53, "key": "literature_id", "value_expression": "literature_id" }, { "line": 54, "key": "title", "value_expression": "title" }, { "line": 55, "key": "authors", "value_expression": "authors" }, { "line": 56, "key": "author_label", "value_expression": "raw_author" }, { "line": 57, "key": "citation_text", "value_expression": "citation_text" }, { "line": 58, "key": "publication_details", "value_expression": "publication_details" }, { "line": 59, "key": "publication_years", "value_expression": "publication_years" }, { "line": 60, "key": "notes", "value_expression": "notes" }, { "line": 61, "key": "source_url", "value_expression": "source_url" }, { "line": 62, "key": "source_page_is_provenance", "value_expression": "True" }, { "line": 70, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 71, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 72, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 73, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 74, "key": "record_type", "value_expression": "'literature_database_entry'" }, { "line": 75, "key": "source", "value_expression": "{**SOURCE_DEFINITION, 'retrieved_at': retrieved_at, 'crawler_version': 'gdo_literature_v1', 'run_id': crawl_run_id or payload.get('crawl_run_id') or ''}" }, { "line": 81, "key": "identifiers", "value_expression": "[{'scheme': 'content_hash', 'value': content_hash, 'role': 'primary', 'source': SOURCE_KEY}, {'scheme': 'synthetic_literature_id', 'value': literature_id, 'role': 'local', 'source': SOURCE_KEY}]" }, { "line": 85, "key": "bibliographic_entry", "value_expression": "literature_entry" }, { "line": 86, "key": "literature", "value_expression": "[literature_entry]" }, { "line": 111, "key": "quality", "value_expression": "{'schema_valid': None, 'completeness': {'has_title': bool(title), 'has_author': bool(authors), 'has_year': bool(publication_years), 'has_notes': bool(notes)}}" }, { "line": 120, "key": "provenance", "value_expression": "{'identity': {'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'content_hash', 'primary_id_value': content_hash, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [document_id]}, 'schema': {'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}, 'normalizer': 'aggregator.gdo_literature_transfer_schema.normalize_gdo_literature_record'}" }, { "line": 139, "key": "extensions", "value_expression": "{'query': query, 'result_index': payload.get('result_index'), 'declared_total_records': payload.get('declared_total_records'), 'year_result_count': payload.get('year_result_count')}" }, { "line": 77, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 78, "key": "crawler_version", "value_expression": "'gdo_literature_v1'" }, { "line": 79, "key": "run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 88, "key": "url", "value_expression": "source_url" }, { "line": 89, "key": "content_type", "value_expression": "'text/html'" }, { "line": 90, "key": "raw_content", "value_expression": "raw_document" }, { "line": 91, "key": "raw_html", "value_expression": "raw_document if '<' in str(raw_document) else ''" }, { "line": 92, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document)" }, { "line": 93, "key": "html_sha256", "value_expression": "provenance.content_sha256(raw_document) if '<' in str(raw_document) else ''" }, { "line": 94, "key": "documents", "value_expression": "[{'document_id': document_id, 'document_kind': 'citation_html', 'url': source_url, 'content_type': 'text/html', 'content_sha256': provenance.content_sha256(raw_document), 'content_size_bytes': len(str(raw_document).encode('utf-8', errors='ignore')), 'retrieved_at': retrieved_at, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'metadata': {'query': query, 'source_page_is_provenance': True}}]" }, { "line": 112, "key": "schema_valid", "value_expression": "None" }, { "line": 113, "key": "completeness", "value_expression": "{'has_title': bool(title), 'has_author': bool(authors), 'has_year': bool(publication_years), 'has_notes': bool(notes)}" }, { "line": 121, "key": "identity", "value_expression": "{'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'content_hash', 'primary_id_value': content_hash, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [document_id]}" }, { "line": 132, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 137, "key": "normalizer", "value_expression": "'aggregator.gdo_literature_transfer_schema.normalize_gdo_literature_record'" }, { "line": 140, "key": "query", "value_expression": "query" }, { "line": 141, "key": "result_index", "value_expression": "payload.get('result_index')" }, { "line": 142, "key": "declared_total_records", "value_expression": "payload.get('declared_total_records')" }, { "line": 143, "key": "year_result_count", "value_expression": "payload.get('year_result_count')" }, { "line": 82, "key": "scheme", "value_expression": "'content_hash'" }, { "line": 82, "key": "value", "value_expression": "content_hash" }, { "line": 82, "key": "role", "value_expression": "'primary'" }, { "line": 82, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 83, "key": "scheme", "value_expression": "'synthetic_literature_id'" }, { "line": 83, "key": "value", "value_expression": "literature_id" }, { "line": 83, "key": "role", "value_expression": "'local'" }, { "line": 83, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 114, "key": "has_title", "value_expression": "bool(title)" }, { "line": 115, "key": "has_author", "value_expression": "bool(authors)" }, { "line": 116, "key": "has_year", "value_expression": "bool(publication_years)" }, { "line": 117, "key": "has_notes", "value_expression": "bool(notes)" }, { "line": 122, "key": "source_id", "value_expression": "provenance.source_id(SOURCE_KEY)" }, { "line": 123, "key": "source_record_id", "value_expression": "source_record_id" }, { "line": 124, "key": "record_id", "value_expression": "source_record_id" }, { "line": 125, "key": "primary_id_scheme", "value_expression": "'content_hash'" }, { "line": 126, "key": "primary_id_value", "value_expression": "content_hash" }, { "line": 127, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 128, "key": "transform_run_id", "value_expression": "transform_run_id or payload.get('transform_run_id') or ''" }, { "line": 129, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 130, "key": "document_ids", "value_expression": "[document_id]" }, { "line": 133, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 134, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 135, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 96, "key": "document_id", "value_expression": "document_id" }, { "line": 97, "key": "document_kind", "value_expression": "'citation_html'" }, { "line": 98, "key": "url", "value_expression": "source_url" }, { "line": 99, "key": "content_type", "value_expression": "'text/html'" }, { "line": 100, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document)" }, { "line": 101, "key": "content_size_bytes", "value_expression": "len(str(raw_document).encode('utf-8', errors='ignore'))" }, { "line": 102, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 103, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 104, "key": "metadata", "value_expression": "{'query': query, 'source_page_is_provenance': True}" }, { "line": 105, "key": "query", "value_expression": "query" }, { "line": 106, "key": "source_page_is_provenance", "value_expression": "True" } ] }, { "symbol": "_identity_text", "line_start": 149, "line_end": 154, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_publication_years", "line_start": 157, "line_end": 163, "literal_get_calls": [ { "line": 158, "key": "publication_years", "expression": "payload.get('publication_years')" } ], "dictionary_entries": [] }, { "symbol": "_split_authors", "line_start": 166, "line_end": 170, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_mapping", "line_start": 173, "line_end": 174, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_text", "line_start": 177, "line_end": 178, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/institution_directory_transfer_schema.py", "sha256": "a069a01be1aab0f7bd196efe85200f900c11add052f5aee2e7fd8674d8163aeb", "functions": [ { "symbol": "normalize_institution_directory_record", "line_start": 20, "line_end": 182, "literal_get_calls": [ { "line": 38, "key": "scraped_at", "expression": "payload.get('scraped_at')" }, { "line": 38, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 39, "key": "directory_url", "expression": "payload.get('directory_url')" }, { "line": 39, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 39, "key": "base_url", "expression": "source_definition.get('base_url')" }, { "line": 32, "key": "link_text", "expression": "payload.get('link_text')" }, { "line": 32, "key": "name", "expression": "payload.get('name')" }, { "line": 33, "key": "website_url", "expression": "payload.get('website_url')" }, { "line": 33, "key": "url", "expression": "payload.get('url')" }, { "line": 34, "key": "country", "expression": "payload.get('country')" }, { "line": 36, "key": "content_sha256", "expression": "payload.get('content_sha256')" }, { "line": 37, "key": "institution_id", "expression": "payload.get('institution_id')" }, { "line": 40, "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 65, "key": "heading_path", "expression": "payload.get('heading_path')" }, { "line": 67, "key": "position", "expression": "payload.get('position')" }, { "line": 46, "key": "name", "expression": "payload.get('name')" }, { "line": 48, "key": "institution_type", "expression": "payload.get('institution_type')" }, { "line": 58, "key": "locality_hint", "expression": "payload.get('locality_hint')" }, { "line": 60, "key": "website_host", "expression": "payload.get('website_host')" }, { "line": 66, "key": "section_id", "expression": "payload.get('section_id')" }, { "line": 69, "key": "raw_link_html", "expression": "payload.get('raw_link_html')" }, { "line": 84, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 163, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 164, "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 177, "key": "raw_data", "expression": "payload.get('raw_data')" }, { "line": 52, "key": "institution_type", "expression": "payload.get('institution_type')" }, { "line": 138, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" } ], "dictionary_entries": [ { "line": 43, "key": "entity_id", "value_expression": "provenance.entity_id(source_key, institution_id, 'institution', 0)" }, { "line": 44, "key": "entity_type", "value_expression": "'institution'" }, { "line": 45, "key": "institution_id", "value_expression": "institution_id" }, { "line": 46, "key": "name", "value_expression": "_clean_text(payload.get('name') or _name_from_label(link_label))" }, { "line": 47, "key": "display_label", "value_expression": "link_label" }, { "line": 48, "key": "institution_type", "value_expression": "_clean_text(payload.get('institution_type') or 'organ_builder')" }, { "line": 49, "key": "institution_family", "value_expression": "'organ_maker'" }, { "line": 50, "key": "roles", "value_expression": "[{'role': _clean_text(payload.get('institution_type') or 'organ_builder'), 'domain': 'organ_making', 'source': source_key}]" }, { "line": 57, "key": "country", "value_expression": "country" }, { "line": 58, "key": "locality_hint", "value_expression": "_clean_text(payload.get('locality_hint') or '')" }, { "line": 59, "key": "website_url", "value_expression": "website_url" }, { "line": 60, "key": "website_host", "value_expression": "_clean_text(payload.get('website_host') or urlsplit(website_url).netloc.lower())" }, { "line": 61, "key": "source_page_is_provenance", "value_expression": "True" }, { "line": 64, "key": "directory_url", "value_expression": "directory_url" }, { "line": 65, "key": "heading_path", "value_expression": "_clean_list(payload.get('heading_path'))" }, { "line": 66, "key": "section_id", "value_expression": "_clean_text(payload.get('section_id') or '')" }, { "line": 67, "key": "position", "value_expression": "_safe_int(payload.get('position'))" }, { "line": 68, "key": "link_text", "value_expression": "link_label" }, { "line": 69, "key": "raw_link_html", "value_expression": "str(payload.get('raw_link_html') or '')" }, { "line": 75, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 76, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 77, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 78, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 79, "key": "record_type", "value_expression": "'institution_directory_entry'" }, { "line": 80, "key": "source", "value_expression": "{**dict(source_definition), 'retrieved_at': retrieved_at, 'crawler_version': f'{source_key}_v1', 'run_id': crawl_run_id or payload.get('crawl_run_id') or ''}" }, { "line": 86, "key": "identifiers", "value_expression": "[{'scheme': 'institution_id', 'value': institution_id, 'role': 'primary', 'source': source_key}, {'scheme': 'normalized_url_hash', 'value': content_hash, 'role': 'derived', 'source': source_key}]" }, { "line": 90, "key": "entity", "value_expression": "institution" }, { "line": 91, "key": "institution", "value_expression": "institution" }, { "line": 92, "key": "directory_context", "value_expression": "directory_context" }, { "line": 93, "key": "links", "value_expression": "[{'entity_id': provenance.entity_id(source_key, institution_id, 'links', 0), 'url': website_url, 'label': link_label, 'relation': 'institution_website', 'source_page_is_provenance': True, 'processor_enrichment': {'eligible': True, 'status': 'candidate', 'task': 'institution_profile_page_analysis', 'target_entity_path': 'institution', 'expected_outputs': ['institution_identity', 'aliases', 'locations', 'institution_roles', 'history_notes', 'related_people_or_firms'], 'requires_review': True, 'fetch_policy': 'public_html_only', 'reason': 'GdO directory entry links to the institution website where richer organ-maker profile evidence may exist.'}}] if website_url else []" }, { "line": 147, "key": "quality", "value_expression": "{'schema_valid': None, 'completeness': {'has_name': bool(institution['name']), 'has_website_url': bool(website_url), 'has_country': bool(country), 'has_locality_hint': bool(institution['locality_hint'])}}" }, { "line": 156, "key": "provenance", "value_expression": "{'identity': {'source_id': provenance.source_id(source_key), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'institution_id', 'primary_id_value': institution_id, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [document_id]}, 'schema': {'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}, 'normalizer': 'aggregator.institution_directory_transfer_schema.normalize_institution_directory_record'}" }, { "line": 175, "key": "extensions", "value_expression": "{source_key: {'raw_data': _clean_mapping(payload.get('raw_data')), 'directory_context': directory_context}}" }, { "line": 82, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 83, "key": "crawler_version", "value_expression": "f'{source_key}_v1'" }, { "line": 84, "key": "run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 122, "key": "url", "value_expression": "directory_url" }, { "line": 123, "key": "source_url", "value_expression": "directory_url" }, { "line": 124, "key": "content_type", "value_expression": "'text/html'" }, { "line": 125, "key": "raw_content", "value_expression": "raw_document" }, { "line": 126, "key": "raw_html", "value_expression": "raw_document if '<' in str(raw_document) else ''" }, { "line": 127, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document)" }, { "line": 128, "key": "html_sha256", "value_expression": "provenance.content_sha256(raw_document) if '<' in str(raw_document) else ''" }, { "line": 129, "key": "documents", "value_expression": "[{'document_id': document_id, 'document_kind': 'directory_link_html', 'url': directory_url, 'content_type': 'text/html', 'content_sha256': provenance.content_sha256(raw_document), 'content_size_bytes': len(str(raw_document).encode('utf-8', errors='ignore')), 'retrieved_at': retrieved_at, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'metadata': {'outbound_url': website_url, 'heading_path': directory_context['heading_path'], 'source_page_is_provenance': True}}]" }, { "line": 148, "key": "schema_valid", "value_expression": "None" }, { "line": 149, "key": "completeness", "value_expression": "{'has_name': bool(institution['name']), 'has_website_url': bool(website_url), 'has_country': bool(country), 'has_locality_hint': bool(institution['locality_hint'])}" }, { "line": 157, "key": "identity", "value_expression": "{'source_id': provenance.source_id(source_key), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'institution_id', 'primary_id_value': institution_id, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [document_id]}" }, { "line": 168, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 173, "key": "normalizer", "value_expression": "'aggregator.institution_directory_transfer_schema.normalize_institution_directory_record'" }, { "line": 52, "key": "role", "value_expression": "_clean_text(payload.get('institution_type') or 'organ_builder')" }, { "line": 53, "key": "domain", "value_expression": "'organ_making'" }, { "line": 54, "key": "source", "value_expression": "source_key" }, { "line": 87, "key": "scheme", "value_expression": "'institution_id'" }, { "line": 87, "key": "value", "value_expression": "institution_id" }, { "line": 87, "key": "role", "value_expression": "'primary'" }, { "line": 87, "key": "source", "value_expression": "source_key" }, { "line": 88, "key": "scheme", "value_expression": "'normalized_url_hash'" }, { "line": 88, "key": "value", "value_expression": "content_hash" }, { "line": 88, "key": "role", "value_expression": "'derived'" }, { "line": 88, "key": "source", "value_expression": "source_key" }, { "line": 150, "key": "has_name", "value_expression": "bool(institution['name'])" }, { "line": 151, "key": "has_website_url", "value_expression": "bool(website_url)" }, { "line": 152, "key": "has_country", "value_expression": "bool(country)" }, { "line": 153, "key": "has_locality_hint", "value_expression": "bool(institution['locality_hint'])" }, { "line": 158, "key": "source_id", "value_expression": "provenance.source_id(source_key)" }, { "line": 159, "key": "source_record_id", "value_expression": "source_record_id" }, { "line": 160, "key": "record_id", "value_expression": "source_record_id" }, { "line": 161, "key": "primary_id_scheme", "value_expression": "'institution_id'" }, { "line": 162, "key": "primary_id_value", "value_expression": "institution_id" }, { "line": 163, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 164, "key": "transform_run_id", "value_expression": "transform_run_id or payload.get('transform_run_id') or ''" }, { "line": 165, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 166, "key": "document_ids", "value_expression": "[document_id]" }, { "line": 169, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 170, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 171, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 177, "key": "raw_data", "value_expression": "_clean_mapping(payload.get('raw_data'))" }, { "line": 178, "key": "directory_context", "value_expression": "directory_context" }, { "line": 95, "key": "entity_id", "value_expression": "provenance.entity_id(source_key, institution_id, 'links', 0)" }, { "line": 96, "key": "url", "value_expression": "website_url" }, { "line": 97, "key": "label", "value_expression": "link_label" }, { "line": 98, "key": "relation", "value_expression": "'institution_website'" }, { "line": 99, "key": "source_page_is_provenance", "value_expression": "True" }, { "line": 100, "key": "processor_enrichment", "value_expression": "{'eligible': True, 'status': 'candidate', 'task': 'institution_profile_page_analysis', 'target_entity_path': 'institution', 'expected_outputs': ['institution_identity', 'aliases', 'locations', 'institution_roles', 'history_notes', 'related_people_or_firms'], 'requires_review': True, 'fetch_policy': 'public_html_only', 'reason': 'GdO directory entry links to the institution website where richer organ-maker profile evidence may exist.'}" }, { "line": 131, "key": "document_id", "value_expression": "document_id" }, { "line": 132, "key": "document_kind", "value_expression": "'directory_link_html'" }, { "line": 133, "key": "url", "value_expression": "directory_url" }, { "line": 134, "key": "content_type", "value_expression": "'text/html'" }, { "line": 135, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document)" }, { "line": 136, "key": "content_size_bytes", "value_expression": "len(str(raw_document).encode('utf-8', errors='ignore'))" }, { "line": 137, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 138, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 139, "key": "metadata", "value_expression": "{'outbound_url': website_url, 'heading_path': directory_context['heading_path'], 'source_page_is_provenance': True}" }, { "line": 101, "key": "eligible", "value_expression": "True" }, { "line": 102, "key": "status", "value_expression": "'candidate'" }, { "line": 103, "key": "task", "value_expression": "'institution_profile_page_analysis'" }, { "line": 104, "key": "target_entity_path", "value_expression": "'institution'" }, { "line": 105, "key": "expected_outputs", "value_expression": "['institution_identity', 'aliases', 'locations', 'institution_roles', 'history_notes', 'related_people_or_firms']" }, { "line": 113, "key": "requires_review", "value_expression": "True" }, { "line": 114, "key": "fetch_policy", "value_expression": "'public_html_only'" }, { "line": 115, "key": "reason", "value_expression": "'GdO directory entry links to the institution website where richer organ-maker profile evidence may exist.'" }, { "line": 140, "key": "outbound_url", "value_expression": "website_url" }, { "line": 141, "key": "heading_path", "value_expression": "directory_context['heading_path']" }, { "line": 142, "key": "source_page_is_provenance", "value_expression": "True" } ] }, { "symbol": "_normalize_url", "line_start": 185, "line_end": 193, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_name_from_label", "line_start": 196, "line_end": 200, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_list", "line_start": 203, "line_end": 207, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_mapping", "line_start": 210, "line_end": 211, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_safe_int", "line_start": 214, "line_end": 220, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_text", "line_start": 223, "line_end": 224, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/ridim_iconography_transfer_schema.py", "sha256": "a78db4720d48770b01031616f17c9b9c65da50549f62b67270e645ca96ffaf80", "functions": [ { "symbol": "normalize_ridim_iconography_record", "line_start": 29, "line_end": 156, "literal_get_calls": [ { "line": 43, "key": "scraped_at", "expression": "payload.get('scraped_at')" }, { "line": 43, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 44, "key": "url", "expression": "payload.get('url')" }, { "line": 44, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 46, "key": "detail_fields", "expression": "payload.get('detail_fields')" }, { "line": 48, "key": "query", "expression": "payload.get('query')" }, { "line": 39, "key": "ridim_id", "expression": "payload.get('ridim_id')" }, { "line": 39, "key": "record_id", "expression": "payload.get('record_id')" }, { "line": 39, "key": "id", "expression": "payload.get('id')" }, { "line": 45, "key": "title", "expression": "payload.get('title')" }, { "line": 47, "key": "raw_html", "expression": "payload.get('raw_html')" }, { "line": 47, "key": "raw_content", "expression": "payload.get('raw_content')" }, { "line": 114, "key": "rights", "expression": "payload.get('rights')" }, { "line": 63, "key": "record_number", "expression": "payload.get('record_number')" }, { "line": 63, "key": "Record ID", "expression": "detail_fields.get('Record ID')" }, { "line": 64, "key": "ridim_siglum", "expression": "payload.get('ridim_siglum')" }, { "line": 64, "key": "RIdIM siglum", "expression": "detail_fields.get('RIdIM siglum')" }, { "line": 65, "key": "object_number", "expression": "payload.get('object_number')" }, { "line": 65, "key": "Object number", "expression": "detail_fields.get('Object number')" }, { "line": 81, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 150, "key": "result_index", "expression": "payload.get('result_index')" }, { "line": 151, "key": "declared_total_records", "expression": "payload.get('declared_total_records')" }, { "line": 152, "key": "search_result_count", "expression": "payload.get('search_result_count')" }, { "line": 119, "key": "creators", "expression": "artwork.get('creators')" }, { "line": 120, "key": "production_date", "expression": "artwork.get('production_date')" }, { "line": 121, "key": "musical_instruments", "expression": "iconography.get('musical_instruments')" }, { "line": 122, "key": "description", "expression": "iconography.get('description')" }, { "line": 123, "key": "images", "expression": "media.get('images')" }, { "line": 133, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 134, "key": "transform_run_id", "expression": "payload.get('transform_run_id')" }, { "line": 148, "key": "rss_item", "expression": "payload.get('rss_item')" }, { "line": 105, "key": "crawl_run_id", "expression": "payload.get('crawl_run_id')" }, { "line": 109, "key": "result_index", "expression": "payload.get('result_index')" }, { "line": 108, "key": "rss_url", "expression": "payload.get('rss_url')" } ], "dictionary_entries": [ { "line": 72, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 73, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 74, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 75, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 76, "key": "record_type", "value_expression": "'music_iconography_record'" }, { "line": 77, "key": "source", "value_expression": "{**SOURCE_DEFINITION, 'retrieved_at': retrieved_at, 'crawler_version': 'ridim_iconography_v1', 'run_id': crawl_run_id or payload.get('crawl_run_id') or ''}" }, { "line": 83, "key": "identifiers", "value_expression": "identifiers" }, { "line": 84, "key": "artwork", "value_expression": "artwork" }, { "line": 85, "key": "holding", "value_expression": "holding" }, { "line": 86, "key": "iconography", "value_expression": "iconography" }, { "line": 87, "key": "media", "value_expression": "media" }, { "line": 88, "key": "links", "value_expression": "links" }, { "line": 114, "key": "rights", "value_expression": "_clean_mapping(payload.get('rights'))" }, { "line": 115, "key": "quality", "value_expression": "{'schema_valid': None, 'completeness': {'has_title': bool(title), 'has_creator': bool(artwork.get('creators')), 'has_production_date': bool(artwork.get('production_date')), 'has_musical_instruments': bool(iconography.get('musical_instruments')), 'has_description': bool(iconography.get('description')), 'has_media_reference': bool(media.get('images'))}}" }, { "line": 126, "key": "provenance", "value_expression": "{'identity': {'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'ridim_id', 'primary_id_value': ridim_id, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [document_id]}, 'schema': {'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}, 'normalizer': 'aggregator.ridim_iconography_transfer_schema.normalize_ridim_iconography_record'}" }, { "line": 145, "key": "extensions", "value_expression": "{'ridim': {'detail_fields': detail_fields, 'rss_item': _clean_mapping(payload.get('rss_item')), 'query': query, 'result_index': payload.get('result_index'), 'declared_total_records': payload.get('declared_total_records'), 'search_result_count': payload.get('search_result_count')}}" }, { "line": 60, "key": "scheme", "value_expression": "'ridim_id'" }, { "line": 60, "key": "value", "value_expression": "ridim_id" }, { "line": 60, "key": "role", "value_expression": "'primary'" }, { "line": 60, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 79, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 80, "key": "crawler_version", "value_expression": "'ridim_iconography_v1'" }, { "line": 81, "key": "run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 90, "key": "url", "value_expression": "source_url" }, { "line": 91, "key": "content_type", "value_expression": "'text/html'" }, { "line": 92, "key": "raw_content", "value_expression": "raw_document" }, { "line": 93, "key": "raw_html", "value_expression": "raw_document if '<' in str(raw_document) else ''" }, { "line": 94, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document)" }, { "line": 95, "key": "html_sha256", "value_expression": "provenance.content_sha256(raw_document) if '<' in str(raw_document) else ''" }, { "line": 96, "key": "documents", "value_expression": "[{'document_id': document_id, 'document_kind': 'detail_html', 'url': source_url, 'content_type': 'text/html', 'content_sha256': provenance.content_sha256(raw_document), 'content_size_bytes': len(str(raw_document).encode('utf-8', errors='ignore')), 'retrieved_at': retrieved_at, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'metadata': {'query': query, 'rss_url': payload.get('rss_url') or '', 'result_index': payload.get('result_index')}}]" }, { "line": 116, "key": "schema_valid", "value_expression": "None" }, { "line": 117, "key": "completeness", "value_expression": "{'has_title': bool(title), 'has_creator': bool(artwork.get('creators')), 'has_production_date': bool(artwork.get('production_date')), 'has_musical_instruments': bool(iconography.get('musical_instruments')), 'has_description': bool(iconography.get('description')), 'has_media_reference': bool(media.get('images'))}" }, { "line": 127, "key": "identity", "value_expression": "{'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': source_record_id, 'record_id': source_record_id, 'primary_id_scheme': 'ridim_id', 'primary_id_value': ridim_id, 'crawl_run_id': crawl_run_id or payload.get('crawl_run_id') or '', 'transform_run_id': transform_run_id or payload.get('transform_run_id') or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': [document_id]}" }, { "line": 138, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 143, "key": "normalizer", "value_expression": "'aggregator.ridim_iconography_transfer_schema.normalize_ridim_iconography_record'" }, { "line": 146, "key": "ridim", "value_expression": "{'detail_fields': detail_fields, 'rss_item': _clean_mapping(payload.get('rss_item')), 'query': query, 'result_index': payload.get('result_index'), 'declared_total_records': payload.get('declared_total_records'), 'search_result_count': payload.get('search_result_count')}" }, { "line": 118, "key": "has_title", "value_expression": "bool(title)" }, { "line": 119, "key": "has_creator", "value_expression": "bool(artwork.get('creators'))" }, { "line": 120, "key": "has_production_date", "value_expression": "bool(artwork.get('production_date'))" }, { "line": 121, "key": "has_musical_instruments", "value_expression": "bool(iconography.get('musical_instruments'))" }, { "line": 122, "key": "has_description", "value_expression": "bool(iconography.get('description'))" }, { "line": 123, "key": "has_media_reference", "value_expression": "bool(media.get('images'))" }, { "line": 128, "key": "source_id", "value_expression": "provenance.source_id(SOURCE_KEY)" }, { "line": 129, "key": "source_record_id", "value_expression": "source_record_id" }, { "line": 130, "key": "record_id", "value_expression": "source_record_id" }, { "line": 131, "key": "primary_id_scheme", "value_expression": "'ridim_id'" }, { "line": 132, "key": "primary_id_value", "value_expression": "ridim_id" }, { "line": 133, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 134, "key": "transform_run_id", "value_expression": "transform_run_id or payload.get('transform_run_id') or ''" }, { "line": 135, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 136, "key": "document_ids", "value_expression": "[document_id]" }, { "line": 139, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 140, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 141, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 147, "key": "detail_fields", "value_expression": "detail_fields" }, { "line": 148, "key": "rss_item", "value_expression": "_clean_mapping(payload.get('rss_item'))" }, { "line": 149, "key": "query", "value_expression": "query" }, { "line": 150, "key": "result_index", "value_expression": "payload.get('result_index')" }, { "line": 151, "key": "declared_total_records", "value_expression": "payload.get('declared_total_records')" }, { "line": 152, "key": "search_result_count", "value_expression": "payload.get('search_result_count')" }, { "line": 69, "key": "scheme", "value_expression": "scheme" }, { "line": 69, "key": "value", "value_expression": "clean_value" }, { "line": 69, "key": "role", "value_expression": "'local'" }, { "line": 69, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 98, "key": "document_id", "value_expression": "document_id" }, { "line": 99, "key": "document_kind", "value_expression": "'detail_html'" }, { "line": 100, "key": "url", "value_expression": "source_url" }, { "line": 101, "key": "content_type", "value_expression": "'text/html'" }, { "line": 102, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_document)" }, { "line": 103, "key": "content_size_bytes", "value_expression": "len(str(raw_document).encode('utf-8', errors='ignore'))" }, { "line": 104, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 105, "key": "crawl_run_id", "value_expression": "crawl_run_id or payload.get('crawl_run_id') or ''" }, { "line": 106, "key": "metadata", "value_expression": "{'query': query, 'rss_url': payload.get('rss_url') or '', 'result_index': payload.get('result_index')}" }, { "line": 107, "key": "query", "value_expression": "query" }, { "line": 108, "key": "rss_url", "value_expression": "payload.get('rss_url') or ''" }, { "line": 109, "key": "result_index", "value_expression": "payload.get('result_index')" } ] }, { "symbol": "canonical_ridim_id", "line_start": 159, "line_end": 169, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_artwork_payload", "line_start": 172, "line_end": 189, "literal_get_calls": [ { "line": 173, "key": "creators", "expression": "payload.get('creators')" }, { "line": 173, "key": "creator", "expression": "payload.get('creator')" }, { "line": 173, "key": "Creator/Producer", "expression": "fields.get('Creator/Producer')" }, { "line": 187, "key": "url", "expression": "payload.get('url')" }, { "line": 187, "key": "source_url", "expression": "payload.get('source_url')" }, { "line": 179, "key": "creator", "expression": "payload.get('creator')" }, { "line": 179, "key": "Creator/Producer", "expression": "fields.get('Creator/Producer')" }, { "line": 180, "key": "production_date", "expression": "payload.get('production_date')" }, { "line": 180, "key": "date", "expression": "payload.get('date')" }, { "line": 180, "key": "Production date", "expression": "fields.get('Production date')" }, { "line": 181, "key": "category", "expression": "payload.get('category')" }, { "line": 181, "key": "Category", "expression": "fields.get('Category')" }, { "line": 182, "key": "object_type", "expression": "payload.get('object_type')" }, { "line": 182, "key": "Object type", "expression": "fields.get('Object type')" }, { "line": 183, "key": "technique", "expression": "payload.get('technique')" }, { "line": 183, "key": "Technique", "expression": "fields.get('Technique')" }, { "line": 184, "key": "materials", "expression": "payload.get('materials')" }, { "line": 184, "key": "Materials", "expression": "fields.get('Materials')" }, { "line": 185, "key": "dimensions", "expression": "payload.get('dimensions')" }, { "line": 185, "key": "Dimensions", "expression": "fields.get('Dimensions')" }, { "line": 186, "key": "description", "expression": "payload.get('description')" }, { "line": 186, "key": "Description", "expression": "fields.get('Description')" } ], "dictionary_entries": [ { "line": 175, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, ridim_id, 'artwork', 0)" }, { "line": 176, "key": "ridim_id", "value_expression": "ridim_id" }, { "line": 177, "key": "title", "value_expression": "title" }, { "line": 178, "key": "creators", "value_expression": "creators" }, { "line": 179, "key": "creator_label", "value_expression": "_clean_text(payload.get('creator') or fields.get('Creator/Producer'))" }, { "line": 180, "key": "production_date", "value_expression": "_clean_text(payload.get('production_date') or payload.get('date') or fields.get('Production date'))" }, { "line": 181, "key": "category", "value_expression": "_clean_text(payload.get('category') or fields.get('Category'))" }, { "line": 182, "key": "object_type", "value_expression": "_clean_text(payload.get('object_type') or fields.get('Object type'))" }, { "line": 183, "key": "technique", "value_expression": "_terms_from_value(payload.get('technique') or fields.get('Technique'))" }, { "line": 184, "key": "materials", "value_expression": "_terms_from_value(payload.get('materials') or fields.get('Materials'))" }, { "line": 185, "key": "dimensions", "value_expression": "_clean_text(payload.get('dimensions') or fields.get('Dimensions'))" }, { "line": 186, "key": "description", "value_expression": "_clean_text(payload.get('description') or fields.get('Description'))" }, { "line": 187, "key": "source_url", "value_expression": "payload.get('url') or payload.get('source_url') or ''" } ] }, { "symbol": "_holding_payload", "line_start": 192, "line_end": 201, "literal_get_calls": [ { "line": 195, "key": "item_location", "expression": "payload.get('item_location')" }, { "line": 195, "key": "Item location", "expression": "fields.get('Item location')" }, { "line": 196, "key": "collection", "expression": "payload.get('collection')" }, { "line": 196, "key": "Collection", "expression": "fields.get('Collection')" }, { "line": 197, "key": "department", "expression": "payload.get('department')" }, { "line": 197, "key": "Department", "expression": "fields.get('Department')" }, { "line": 198, "key": "object_number", "expression": "payload.get('object_number')" }, { "line": 198, "key": "Object number", "expression": "fields.get('Object number')" }, { "line": 199, "key": "ridim_siglum", "expression": "payload.get('ridim_siglum')" }, { "line": 199, "key": "RIdIM siglum", "expression": "fields.get('RIdIM siglum')" } ], "dictionary_entries": [ { "line": 195, "key": "item_location", "value_expression": "_clean_text(payload.get('item_location') or fields.get('Item location'))" }, { "line": 196, "key": "collection", "value_expression": "_clean_text(payload.get('collection') or fields.get('Collection'))" }, { "line": 197, "key": "department", "value_expression": "_clean_text(payload.get('department') or fields.get('Department'))" }, { "line": 198, "key": "object_number", "value_expression": "_clean_text(payload.get('object_number') or fields.get('Object number'))" }, { "line": 199, "key": "ridim_siglum", "value_expression": "_clean_text(payload.get('ridim_siglum') or fields.get('RIdIM siglum'))" } ] }, { "symbol": "_iconography_payload", "line_start": 204, "line_end": 216, "literal_get_calls": [ { "line": 205, "key": "iconclass", "expression": "payload.get('iconclass')" }, { "line": 205, "key": "Iconclass", "expression": "fields.get('Iconclass')" }, { "line": 206, "key": "subjects", "expression": "payload.get('subjects')" }, { "line": 206, "key": "Subjects", "expression": "fields.get('Subjects')" }, { "line": 207, "key": "musical_instruments", "expression": "payload.get('musical_instruments')" }, { "line": 207, "key": "Musical instruments", "expression": "fields.get('Musical instruments')" }, { "line": 213, "key": "description", "expression": "payload.get('description')" }, { "line": 213, "key": "Description", "expression": "fields.get('Description')" } ], "dictionary_entries": [ { "line": 210, "key": "iconclass", "value_expression": "iconclass" }, { "line": 211, "key": "subjects", "value_expression": "subjects" }, { "line": 212, "key": "musical_instruments", "value_expression": "instruments" }, { "line": 213, "key": "description", "value_expression": "_clean_text(payload.get('description') or fields.get('Description'))" }, { "line": 214, "key": "normalized_instrument_terms", "value_expression": "[_normalize_term(item) for item in instruments]" } ] }, { "symbol": "_media_payload", "line_start": 219, "line_end": 241, "literal_get_calls": [ { "line": 221, "key": "images", "expression": "payload.get('images')" }, { "line": 224, "key": "url", "expression": "image.get('url')" }, { "line": 224, "key": "link", "expression": "image.get('link')" }, { "line": 233, "key": "file_name", "expression": "image.get('file_name')" }, { "line": 234, "key": "label", "expression": "image.get('label')" }, { "line": 234, "key": "title", "expression": "image.get('title')" }, { "line": 235, "key": "content_type", "expression": "image.get('content_type')" }, { "line": 236, "key": "status", "expression": "image.get('status')" } ], "dictionary_entries": [ { "line": 241, "key": "images", "value_expression": "images" }, { "line": 241, "key": "audio", "value_expression": "[]" }, { "line": 241, "key": "videos", "value_expression": "[]" }, { "line": 241, "key": "documents", "value_expression": "[]" }, { "line": 230, "key": "entity_id", "value_expression": "provenance.entity_id(SOURCE_KEY, ridim_id, 'image', index)" }, { "line": 231, "key": "kind", "value_expression": "'image'" }, { "line": 232, "key": "url", "value_expression": "url" }, { "line": 233, "key": "file_name", "value_expression": "image.get('file_name') or url.rsplit('/', 1)[-1].split('?', 1)[0]" }, { "line": 234, "key": "label", "value_expression": "image.get('label') or image.get('title') or ''" }, { "line": 235, "key": "content_type", "value_expression": "image.get('content_type') or ''" }, { "line": 236, "key": "status", "value_expression": "image.get('status') or 'discovered'" }, { "line": 237, "key": "source_page_url", "value_expression": "source_url" } ] }, { "symbol": "_links_payload", "line_start": 244, "line_end": 249, "literal_get_calls": [ { "line": 246, "key": "canonical_url", "expression": "payload.get('canonical_url')" } ], "dictionary_entries": [ { "line": 245, "key": "url", "value_expression": "source_url" }, { "line": 245, "key": "label", "value_expression": "'RIdIM record'" }, { "line": 245, "key": "role", "value_expression": "'source_record'" }, { "line": 248, "key": "url", "value_expression": "canonical_url" }, { "line": 248, "key": "label", "value_expression": "'RIdIM canonical record'" }, { "line": 248, "key": "role", "value_expression": "'canonical'" } ] }, { "symbol": "_identity_text", "line_start": 252, "line_end": 257, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_terms_from_value", "line_start": 260, "line_end": 265, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_normalize_term", "line_start": 268, "line_end": 270, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_absolute_url", "line_start": 273, "line_end": 277, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_drop_empty", "line_start": 280, "line_end": 281, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_mapping", "line_start": 284, "line_end": 285, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_text", "line_start": 288, "line_end": 289, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/forum_transfer_schema.py", "sha256": "19bbe595bd263c5393dbceb9c3c369e0f9a26ede9a65f09934b0213cd4fa3de8", "functions": [ { "symbol": "canonical_source_key", "line_start": 104, "line_end": 112, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "empty_forum_record", "line_start": 115, "line_end": 119, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "normalize_forum_thread_record", "line_start": 122, "line_end": 165, "literal_get_calls": [ { "line": 140, "key": "source_info", "expression": "payload.get('source_info')" }, { "line": 140, "key": "source_info", "expression": "payload.get('source_info')" } ], "dictionary_entries": [] }, { "symbol": "_apply_source", "line_start": 168, "line_end": 179, "literal_get_calls": [ { "line": 172, "key": "name", "expression": "metadata.get('name')" }, { "line": 172, "key": "name", "expression": "source.get('name')" }, { "line": 173, "key": "base_url", "expression": "metadata.get('base_url')" }, { "line": 173, "key": "base_url", "expression": "source.get('base_url', '')" }, { "line": 176, "key": "retrieved_at", "expression": "source_info.get('retrieved_at')" }, { "line": 177, "key": "retrieval_date", "expression": "source_info.get('retrieval_date')" }, { "line": 178, "key": "retrieved_at", "expression": "source.get('retrieved_at', '')" } ], "dictionary_entries": [] }, { "symbol": "_merge_known_roots", "line_start": 182, "line_end": 190, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_normalize_thread", "line_start": 193, "line_end": 212, "literal_get_calls": [ { "line": 201, "key": "thread_id", "expression": "thread.get('thread_id')" }, { "line": 201, "key": "thread_id", "expression": "payload.get('thread_id')" }, { "line": 204, "key": "url", "expression": "thread.get('url')" }, { "line": 204, "key": "url", "expression": "payload.get('url')" }, { "line": 205, "key": "posts", "expression": "record.get('posts')" }, { "line": 205, "key": "posts", "expression": "record.get('posts')" }, { "line": 206, "key": "post_count", "expression": "thread.get('post_count')" }, { "line": 210, "key": "created_at", "expression": "thread.get('created_at')" }, { "line": 210, "key": "created_at", "expression": "first_post.get('created_at', '')" }, { "line": 211, "key": "updated_at", "expression": "thread.get('updated_at')" }, { "line": 211, "key": "created_at", "expression": "last_post.get('created_at', '')" }, { "line": 212, "key": "starter", "expression": "thread.get('starter')" }, { "line": 212, "key": "author_display_name", "expression": "first_post.get('author_display_name', '')" } ], "dictionary_entries": [] }, { "symbol": "_normalize_lists", "line_start": 215, "line_end": 223, "literal_get_calls": [ { "line": 221, "key": "images", "expression": "media.get('images', [])" }, { "line": 222, "key": "url", "expression": "item.get('url')" }, { "line": 222, "key": "link", "expression": "item.get('link')" } ], "dictionary_entries": [] }, { "symbol": "_normalize_source_snapshot", "line_start": 226, "line_end": 248, "literal_get_calls": [ { "line": 235, "key": "source_info", "expression": "payload.get('source_info')" }, { "line": 236, "key": "url", "expression": "snapshot.get('url')" }, { "line": 236, "key": "url", "expression": "source_info.get('url')" }, { "line": 236, "key": "url", "expression": "payload.get('url')" }, { "line": 237, "key": "page_urls", "expression": "snapshot.get('page_urls')" }, { "line": 240, "key": "content_type", "expression": "snapshot.get('content_type')" }, { "line": 241, "key": "content_type", "expression": "snapshot.get('content_type')" }, { "line": 246, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 248, "key": "documents", "expression": "snapshot.get('documents')" }, { "line": 235, "key": "source_info", "expression": "payload.get('source_info')" }, { "line": 238, "key": "raw_content", "expression": "snapshot.get('raw_content')" }, { "line": 238, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 238, "key": "html", "expression": "payload.get('html')" }, { "line": 242, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 245, "key": "content_sha256", "expression": "snapshot.get('content_sha256')" }, { "line": 247, "key": "html_sha256", "expression": "snapshot.get('html_sha256')" } ], "dictionary_entries": [] }, { "symbol": "_normalize_identifiers", "line_start": 251, "line_end": 267, "literal_get_calls": [ { "line": 264, "key": "forum_id", "expression": "record.get('forum', {}).get('forum_id')" }, { "line": 252, "key": "identifiers", "expression": "record.get('identifiers')" }, { "line": 253, "key": "thread_id", "expression": "record.get('thread', {}).get('thread_id')" }, { "line": 264, "key": "forum", "expression": "record.get('forum', {})" }, { "line": 253, "key": "thread", "expression": "record.get('thread', {})" }, { "line": 258, "key": "primary_identifier", "expression": "SOURCE_METADATA.get(key, {}).get('primary_identifier', 'thread_id')" } ], "dictionary_entries": [ { "line": 258, "key": "scheme", "value_expression": "SOURCE_METADATA.get(key, {}).get('primary_identifier', 'thread_id')" }, { "line": 259, "key": "value", "value_expression": "str(thread_id)" }, { "line": 260, "key": "role", "value_expression": "'primary'" }, { "line": 261, "key": "source", "value_expression": "key" }, { "line": 266, "key": "scheme", "value_expression": "'forum_id'" }, { "line": 266, "key": "value", "value_expression": "str(forum_id)" }, { "line": 266, "key": "role", "value_expression": "'context'" }, { "line": 266, "key": "source", "value_expression": "key" } ] }, { "symbol": "_apply_provenance_identity", "line_start": 270, "line_end": 292, "literal_get_calls": [ { "line": 284, "key": "crawl_run_id", "expression": "identity.get('crawl_run_id', '')" }, { "line": 285, "key": "transform_run_id", "expression": "identity.get('transform_run_id', '')" }, { "line": 286, "key": "artifact_ids", "expression": "identity.get('artifact_ids')" } ], "dictionary_entries": [] }, { "symbol": "_record_identity_parts", "line_start": 295, "line_end": 307, "literal_get_calls": [ { "line": 298, "key": "identifiers", "expression": "record.get('identifiers')" }, { "line": 301, "key": "thread_id", "expression": "(record.get('thread') or {}).get('thread_id')" }, { "line": 304, "key": "url", "expression": "(record.get('source_snapshot') or {}).get('url')" }, { "line": 297, "key": "primary_identifier", "expression": "SOURCE_METADATA.get(key, {}).get('primary_identifier', 'thread_id')" }, { "line": 301, "key": "thread", "expression": "record.get('thread')" }, { "line": 304, "key": "source_snapshot", "expression": "record.get('source_snapshot')" }, { "line": 299, "key": "role", "expression": "item.get('role')" }, { "line": 299, "key": "value", "expression": "item.get('value')" }, { "line": 300, "key": "value", "expression": "item.get('value')" }, { "line": 301, "key": "thread", "expression": "record.get('thread')" }, { "line": 304, "key": "source_snapshot", "expression": "record.get('source_snapshot')" }, { "line": 300, "key": "scheme", "expression": "item.get('scheme')" } ], "dictionary_entries": [] }, { "symbol": "_apply_document_identity", "line_start": 310, "line_end": 328, "literal_get_calls": [ { "line": 313, "key": "raw_html", "expression": "snapshot.get('raw_html')" }, { "line": 312, "key": "documents", "expression": "snapshot.get('documents')" }, { "line": 328, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 322, "key": "url", "expression": "snapshot.get('url', '')" }, { "line": 323, "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at', '')" }, { "line": 324, "key": "crawl_run_id", "expression": "identity.get('crawl_run_id', '')" }, { "line": 328, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 320, "key": "html_sha256", "expression": "snapshot.get('html_sha256')" }, { "line": 323, "key": "source", "expression": "record.get('source', {})" } ], "dictionary_entries": [ { "line": 318, "key": "document_id", "value_expression": "provenance.document_id(key, primary_id, 'raw_html')" }, { "line": 319, "key": "document_kind", "value_expression": "'raw_html'" }, { "line": 320, "key": "content_sha256", "value_expression": "snapshot.get('html_sha256') or provenance.content_sha256(raw_html)" }, { "line": 321, "key": "content_type", "value_expression": "'text/html'" }, { "line": 322, "key": "url", "value_expression": "snapshot.get('url', '')" }, { "line": 323, "key": "retrieved_at", "value_expression": "record.get('source', {}).get('retrieved_at', '')" }, { "line": 324, "key": "crawl_run_id", "value_expression": "identity.get('crawl_run_id', '')" } ] }, { "symbol": "_apply_entity_ids", "line_start": 331, "line_end": 350, "literal_get_calls": [ { "line": 332, "key": "key", "expression": "record.get('source', {}).get('key', '')" }, { "line": 332, "key": "source", "expression": "record.get('source', {})" }, { "line": 335, "key": "posts", "expression": "record.get('posts')" }, { "line": 338, "key": "participants", "expression": "record.get('participants')" }, { "line": 341, "key": "interactions", "expression": "record.get('interactions')" }, { "line": 344, "key": "links", "expression": "record.get('links')" }, { "line": 337, "key": "post_id", "expression": "item.get('post_id')" }, { "line": 340, "key": "display_name", "expression": "item.get('display_name')" }, { "line": 346, "key": "url", "expression": "item.get('url')" }, { "line": 348, "key": "media", "expression": "record.get('media', {})" }, { "line": 350, "key": "url", "expression": "item.get('url')" }, { "line": 350, "key": "link", "expression": "item.get('link')" } ], "dictionary_entries": [] }, { "symbol": "_attach_extensions", "line_start": 353, "line_end": 358, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_as_list", "line_start": 361, "line_end": 368, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_append_unique", "line_start": 371, "line_end": 373, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_deep_merge", "line_start": 376, "line_end": 383, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/vmi_forum_transfer_schema.py", "sha256": "36bd22234798c8458672138a454ad8b316f60f4a5ea79eda01126302360aa1a5", "functions": [ { "symbol": "normalize_vmi_forum_post_record", "line_start": 76, "line_end": 155, "literal_get_calls": [ { "line": 100, "key": "extracted_mentions", "expression": "payload.get('extracted_mentions')" }, { "line": 94, "key": "retrieved_at", "expression": "payload.get('retrieved_at')" }, { "line": 95, "key": "post_url", "expression": "post.get('post_url')" }, { "line": 95, "key": "url", "expression": "post.get('url')" }, { "line": 95, "key": "url", "expression": "payload.get('url')" }, { "line": 95, "key": "thread_url", "expression": "thread.get('thread_url')" }, { "line": 89, "key": "post", "expression": "payload.get('post')" }, { "line": 90, "key": "thread", "expression": "payload.get('thread')" }, { "line": 91, "key": "forum", "expression": "payload.get('forum')" }, { "line": 92, "key": "post_id", "expression": "post.get('post_id')" }, { "line": 92, "key": "post_id", "expression": "payload.get('post_id')" }, { "line": 93, "key": "thread_id", "expression": "thread.get('thread_id')" }, { "line": 93, "key": "thread_id", "expression": "payload.get('thread_id')" }, { "line": 96, "key": "html", "expression": "post.get('html')" }, { "line": 96, "key": "html", "expression": "payload.get('html')" }, { "line": 97, "key": "text", "expression": "post.get('text')" }, { "line": 97, "key": "text", "expression": "payload.get('text')" }, { "line": 142, "key": "quality", "expression": "payload.get('quality')" }, { "line": 153, "key": "document_id", "expression": "doc.get('document_id')" }, { "line": 129, "key": "author", "expression": "post.get('author')" }, { "line": 129, "key": "author", "expression": "payload.get('author')" }, { "line": 141, "key": "moderation", "expression": "payload.get('moderation')" }, { "line": 150, "key": "extensions", "expression": "payload.get('extensions')" }, { "line": 98, "key": "links", "expression": "payload.get('links')" }, { "line": 98, "key": "links", "expression": "post.get('links')" }, { "line": 99, "key": "media", "expression": "payload.get('media')" }, { "line": 105, "key": "title", "expression": "thread.get('title')" }, { "line": 108, "key": "author", "expression": "post.get('author')" }, { "line": 108, "key": "author", "expression": "payload.get('author')" }, { "line": 129, "key": "author", "expression": "post.get('author')" }, { "line": 108, "key": "author", "expression": "post.get('author')" } ], "dictionary_entries": [ { "line": 112, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 113, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 114, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 115, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 116, "key": "record_type", "value_expression": "RECORD_TYPE" }, { "line": 117, "key": "source", "value_expression": "_source_payload(key, retrieved_at, crawl_run_id)" }, { "line": 118, "key": "identifiers", "value_expression": "[{'scheme': 'post_id', 'value': post_id, 'role': 'primary', 'source': key}, *([{'scheme': 'thread_id', 'value': thread_id, 'role': 'context', 'source': key}] if thread_id else [])]" }, { "line": 126, "key": "forum", "value_expression": "_forum_payload(forum)" }, { "line": 127, "key": "thread", "value_expression": "_thread_payload(thread)" }, { "line": 128, "key": "post", "value_expression": "_post_payload(post, post_id=post_id, post_url=post_url, text=text, html=html)" }, { "line": 129, "key": "author", "value_expression": "_author_payload(post.get('author') if isinstance(post.get('author'), Mapping) else payload.get('author'))" }, { "line": 130, "key": "source_snapshot", "value_expression": "_source_snapshot(key=key, post_id=post_id, url=post_url, html=html, retrieved_at=retrieved_at, crawl_run_id=crawl_run_id)" }, { "line": 138, "key": "extracted_mentions", "value_expression": "_with_entity_ids(mentions, key, post_id, 'extracted_mentions')" }, { "line": 139, "key": "links", "value_expression": "_with_entity_ids(links, key, post_id, 'links')" }, { "line": 140, "key": "media", "value_expression": "_with_entity_ids(media, key, post_id, 'media')" }, { "line": 141, "key": "moderation", "value_expression": "dict(payload.get('moderation') or {})" }, { "line": 142, "key": "quality", "value_expression": "_quality_payload(payload.get('quality'), mentions)" }, { "line": 143, "key": "provenance", "value_expression": "_provenance_payload(key=key, post_id=post_id, crawl_run_id=crawl_run_id, transform_run_id=transform_run_id, artifact_ids=artifact_ids)" }, { "line": 150, "key": "extensions", "value_expression": "dict(payload.get('extensions') or {})" }, { "line": 119, "key": "scheme", "value_expression": "'post_id'" }, { "line": 119, "key": "value", "value_expression": "post_id" }, { "line": 119, "key": "role", "value_expression": "'primary'" }, { "line": 119, "key": "source", "value_expression": "key" }, { "line": 121, "key": "scheme", "value_expression": "'thread_id'" }, { "line": 121, "key": "value", "value_expression": "thread_id" }, { "line": 121, "key": "role", "value_expression": "'context'" }, { "line": 121, "key": "source", "value_expression": "key" } ] }, { "symbol": "extract_candidate_mentions", "line_start": 158, "line_end": 244, "literal_get_calls": [ { "line": 232, "key": "normalized_url", "expression": "product_links[0].get('normalized_url')" }, { "line": 186, "key": "is_forum_evidence_link", "expression": "link.get('is_forum_evidence_link')" }, { "line": 186, "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 186, "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 233, "key": "normalized_url", "expression": "link.get('normalized_url')" }, { "line": 233, "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" } ], "dictionary_entries": [ { "line": 172, "key": "pipe_organ_terms", "value_expression": "_matched_terms(text, PIPE_ORGAN_TERMS)" }, { "line": 173, "key": "engine_terms", "value_expression": "_matched_terms(text, ENGINE_TERMS)" }, { "line": 174, "key": "version_terms", "value_expression": "VERSION_RE.findall(text)" }, { "line": 175, "key": "availability_terms", "value_expression": "_matched_terms(text, ('free', 'available', 'released', 'sale', 'discount'))" }, { "line": 176, "key": "price_terms", "value_expression": "_matched_terms(text, PRICE_TERMS)" }, { "line": 177, "key": "download_terms", "value_expression": "_matched_terms(text, DOWNLOAD_TERMS)" }, { "line": 178, "key": "evidence_strength", "value_expression": "evidence_strength" }, { "line": 215, "key": "mention_id", "value_expression": "f'ment:{source_key}:post:{post_id}:0'" }, { "line": 216, "key": "kind", "value_expression": "'virtual_musical_instrument'" }, { "line": 217, "key": "raw_label", "value_expression": "title" }, { "line": 218, "key": "normalized_label", "value_expression": "title" }, { "line": 219, "key": "context_excerpt", "value_expression": "_excerpt(text)" }, { "line": 220, "key": "source_selector", "value_expression": "{'type': 'css', 'selector': f'article[data-content=\"post-{post_id}\"] .bbWrapper', 'quote_hash': provenance.content_sha256(_excerpt(text))}" }, { "line": 225, "key": "confidence", "value_expression": "round(confidence, 2)" }, { "line": 226, "key": "signals", "value_expression": "signals" }, { "line": 221, "key": "type", "value_expression": "'css'" }, { "line": 222, "key": "selector", "value_expression": "f'article[data-content=\"post-{post_id}\"] .bbWrapper'" }, { "line": 223, "key": "quote_hash", "value_expression": "provenance.content_sha256(_excerpt(text))" }, { "line": 228, "key": "title", "value_expression": "title" }, { "line": 229, "key": "version_label", "value_expression": "f'v{version_label}' if version_label and (not str(version_label).lower().startswith('v')) else version_label" }, { "line": 230, "key": "developer_label", "value_expression": "developer_label" }, { "line": 231, "key": "publisher_label", "value_expression": "developer_label if _looks_like_publisher_announcement(author, post_text) else None" }, { "line": 232, "key": "homepage_url", "value_expression": "product_links[0].get('normalized_url') if product_links else None" }, { "line": 233, "key": "download_url", "value_expression": "next((link.get('normalized_url') for link in product_links if link.get('is_download_candidate')), None)" }, { "line": 234, "key": "engine_labels", "value_expression": "signals['engine_terms']" }, { "line": 235, "key": "platform_labels", "value_expression": "[]" }, { "line": 236, "key": "family_hint", "value_expression": "'pipe_organ' if signals['pipe_organ_terms'] else None" }, { "line": 237, "key": "production_method_hint", "value_expression": "'sampled' if 'sample set' in lower or 'sample library' in lower else None" }, { "line": 238, "key": "pipe_organ_relation_hint", "value_expression": "relation_hint" }, { "line": 239, "key": "represented_organ_label", "value_expression": "represented_organ_label" }, { "line": 240, "key": "institution_label", "value_expression": "institution_label" }, { "line": 241, "key": "person_label", "value_expression": "person_label" } ] }, { "symbol": "_best_candidate_title", "line_start": 247, "line_end": 265, "literal_get_calls": [ { "line": 259, "key": "link_text", "expression": "link.get('link_text')" } ], "dictionary_entries": [] }, { "symbol": "_trim_title_boundary", "line_start": 268, "line_end": 276, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_is_recommendation_thread_title", "line_start": 279, "line_end": 281, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_is_generic_candidate_title", "line_start": 284, "line_end": 286, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_developer_label", "line_start": 289, "line_end": 302, "literal_get_calls": [ { "line": 298, "key": "display_name", "expression": "author.get('display_name')" }, { "line": 298, "key": "name", "expression": "author.get('name')" }, { "line": 299, "key": "role_label", "expression": "author.get('role_label')" }, { "line": 299, "key": "title", "expression": "author.get('title')" } ], "dictionary_entries": [] }, { "symbol": "_person_label", "line_start": 305, "line_end": 310, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_represented_organ_label", "line_start": 313, "line_end": 322, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_institution_label", "line_start": 325, "line_end": 336, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_pipe_organ_relation_hint", "line_start": 339, "line_end": 347, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_evidence_strength", "line_start": 350, "line_end": 366, "literal_get_calls": [ { "line": 354, "key": "is_product_homepage_candidate", "expression": "link.get('is_product_homepage_candidate')" }, { "line": 354, "key": "is_download_candidate", "expression": "link.get('is_download_candidate')" }, { "line": 351, "key": "role_label", "expression": "author.get('role_label')" }, { "line": 351, "key": "title", "expression": "author.get('title')" }, { "line": 356, "key": "is_forum_evidence_link", "expression": "link.get('is_forum_evidence_link')" } ], "dictionary_entries": [] }, { "symbol": "_looks_like_publisher_announcement", "line_start": 369, "line_end": 372, "literal_get_calls": [ { "line": 370, "key": "role_label", "expression": "author.get('role_label')" }, { "line": 370, "key": "title", "expression": "author.get('title')" } ], "dictionary_entries": [] }, { "symbol": "_clean_entity_label", "line_start": 375, "line_end": 380, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "canonical_source_key", "line_start": 383, "line_end": 387, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_source_payload", "line_start": 390, "line_end": 400, "literal_get_calls": [], "dictionary_entries": [ { "line": 392, "key": "key", "value_expression": "key" }, { "line": 393, "key": "name", "value_expression": "'VI-Control'" }, { "line": 394, "key": "display_name", "value_expression": "'VI-Control'" }, { "line": 395, "key": "base_url", "value_expression": "BASE_URL if key == SOURCE_KEY else ''" }, { "line": 396, "key": "type", "value_expression": "'forum'" }, { "line": 397, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 398, "key": "crawler_version", "value_expression": "'aggregator.vi_control_forum_crawler/1.0.0'" }, { "line": 399, "key": "run_id", "value_expression": "crawl_run_id or ''" } ] }, { "symbol": "_forum_payload", "line_start": 403, "line_end": 415, "literal_get_calls": [ { "line": 405, "key": "forum_platform", "expression": "forum.get('forum_platform')" }, { "line": 406, "key": "locale", "expression": "forum.get('locale')" }, { "line": 407, "key": "timezone", "expression": "forum.get('timezone')" }, { "line": 408, "key": "public_access", "expression": "forum.get('public_access', True)" }, { "line": 409, "key": "credential_required", "expression": "forum.get('credential_required', False)" }, { "line": 410, "key": "terms_url", "expression": "forum.get('terms_url')" }, { "line": 411, "key": "robots_status", "expression": "forum.get('robots_status')" }, { "line": 413, "key": "category_label", "expression": "forum.get('category_label')" }, { "line": 413, "key": "forum_title", "expression": "forum.get('forum_title')" }, { "line": 414, "key": "category_url", "expression": "forum.get('category_url')" }, { "line": 412, "key": "category_id", "expression": "forum.get('category_id')" } ], "dictionary_entries": [ { "line": 405, "key": "forum_platform", "value_expression": "forum.get('forum_platform') or 'XenForo'" }, { "line": 406, "key": "locale", "value_expression": "forum.get('locale') or 'en'" }, { "line": 407, "key": "timezone", "value_expression": "forum.get('timezone') or ''" }, { "line": 408, "key": "public_access", "value_expression": "bool(forum.get('public_access', True))" }, { "line": 409, "key": "credential_required", "value_expression": "bool(forum.get('credential_required', False))" }, { "line": 410, "key": "terms_url", "value_expression": "forum.get('terms_url') or ''" }, { "line": 411, "key": "robots_status", "value_expression": "forum.get('robots_status') or 'not_checked'" }, { "line": 412, "key": "category_id", "value_expression": "str(forum.get('category_id') or '')" }, { "line": 413, "key": "category_label", "value_expression": "forum.get('category_label') or forum.get('forum_title') or ''" }, { "line": 414, "key": "category_url", "value_expression": "forum.get('category_url') or ''" } ] }, { "symbol": "_thread_payload", "line_start": 418, "line_end": 435, "literal_get_calls": [ { "line": 427, "key": "page_number", "expression": "thread.get('page_number')" }, { "line": 428, "key": "reply_count", "expression": "thread.get('reply_count')" }, { "line": 429, "key": "view_count", "expression": "thread.get('view_count')" }, { "line": 421, "key": "thread_url", "expression": "thread.get('thread_url')" }, { "line": 421, "key": "url", "expression": "thread.get('url')" }, { "line": 422, "key": "canonical_thread_url", "expression": "thread.get('canonical_thread_url')" }, { "line": 422, "key": "thread_url", "expression": "thread.get('thread_url')" }, { "line": 422, "key": "url", "expression": "thread.get('url')" }, { "line": 423, "key": "title", "expression": "thread.get('title')" }, { "line": 425, "key": "category_label", "expression": "thread.get('category_label')" }, { "line": 430, "key": "created_at", "expression": "thread.get('created_at')" }, { "line": 431, "key": "updated_at", "expression": "thread.get('updated_at')" }, { "line": 432, "key": "is_locked", "expression": "thread.get('is_locked', False)" }, { "line": 433, "key": "is_sticky", "expression": "thread.get('is_sticky', False)" }, { "line": 434, "key": "is_deleted", "expression": "thread.get('is_deleted', False)" }, { "line": 420, "key": "thread_id", "expression": "thread.get('thread_id')" }, { "line": 424, "key": "category_id", "expression": "thread.get('category_id')" }, { "line": 426, "key": "tags", "expression": "thread.get('tags')" } ], "dictionary_entries": [ { "line": 420, "key": "thread_id", "value_expression": "str(thread.get('thread_id') or '')" }, { "line": 421, "key": "thread_url", "value_expression": "thread.get('thread_url') or thread.get('url') or ''" }, { "line": 422, "key": "canonical_thread_url", "value_expression": "thread.get('canonical_thread_url') or thread.get('thread_url') or thread.get('url') or ''" }, { "line": 423, "key": "title", "value_expression": "thread.get('title') or ''" }, { "line": 424, "key": "category_id", "value_expression": "str(thread.get('category_id') or '')" }, { "line": 425, "key": "category_label", "value_expression": "thread.get('category_label') or ''" }, { "line": 426, "key": "tags", "value_expression": "list(thread.get('tags') or [])" }, { "line": 427, "key": "page_number", "value_expression": "thread.get('page_number')" }, { "line": 428, "key": "reply_count", "value_expression": "thread.get('reply_count')" }, { "line": 429, "key": "view_count", "value_expression": "thread.get('view_count')" }, { "line": 430, "key": "created_at", "value_expression": "thread.get('created_at') or ''" }, { "line": 431, "key": "updated_at", "value_expression": "thread.get('updated_at') or ''" }, { "line": 432, "key": "is_locked", "value_expression": "bool(thread.get('is_locked', False))" }, { "line": 433, "key": "is_sticky", "value_expression": "bool(thread.get('is_sticky', False))" }, { "line": 434, "key": "is_deleted", "value_expression": "bool(thread.get('is_deleted', False))" } ] }, { "symbol": "_post_payload", "line_start": 438, "line_end": 455, "literal_get_calls": [ { "line": 442, "key": "position_in_thread", "expression": "post.get('position_in_thread')" }, { "line": 443, "key": "posted_at", "expression": "post.get('posted_at')" }, { "line": 443, "key": "created_at", "expression": "post.get('created_at')" }, { "line": 444, "key": "edited_at", "expression": "post.get('edited_at')" }, { "line": 445, "key": "title", "expression": "post.get('title')" }, { "line": 451, "key": "selectors", "expression": "post.get('selectors')" }, { "line": 449, "key": "quoted_post_ids", "expression": "post.get('quoted_post_ids')" }, { "line": 450, "key": "mentions", "expression": "post.get('mentions')" } ], "dictionary_entries": [ { "line": 440, "key": "post_id", "value_expression": "post_id" }, { "line": 441, "key": "post_url", "value_expression": "post_url" }, { "line": 442, "key": "position_in_thread", "value_expression": "post.get('position_in_thread')" }, { "line": 443, "key": "posted_at", "value_expression": "post.get('posted_at') or post.get('created_at') or ''" }, { "line": 444, "key": "edited_at", "value_expression": "post.get('edited_at') or ''" }, { "line": 445, "key": "title", "value_expression": "post.get('title') or ''" }, { "line": 446, "key": "raw_html_hash", "value_expression": "provenance.content_sha256(html)" }, { "line": 447, "key": "text_hash", "value_expression": "provenance.content_sha256(text)" }, { "line": 448, "key": "text_excerpt", "value_expression": "_excerpt(text)" }, { "line": 449, "key": "quoted_post_ids", "value_expression": "list(post.get('quoted_post_ids') or [])" }, { "line": 450, "key": "mentions", "value_expression": "list(post.get('mentions') or [])" }, { "line": 451, "key": "selectors", "value_expression": "post.get('selectors') or {'post': f'article[data-content=\"post-{post_id}\"]', 'content': f'article[data-content=\"post-{post_id}\"] .bbWrapper'}" }, { "line": 452, "key": "post", "value_expression": "f'article[data-content=\"post-{post_id}\"]'" }, { "line": 453, "key": "content", "value_expression": "f'article[data-content=\"post-{post_id}\"] .bbWrapper'" } ] }, { "symbol": "_author_payload", "line_start": 458, "line_end": 466, "literal_get_calls": [ { "line": 460, "key": "source_author_id", "expression": "author.get('source_author_id')" }, { "line": 460, "key": "id", "expression": "author.get('id')" }, { "line": 463, "key": "display_name", "expression": "author.get('display_name')" }, { "line": 463, "key": "name", "expression": "author.get('name')" }, { "line": 464, "key": "profile_url", "expression": "author.get('profile_url')" }, { "line": 464, "key": "link", "expression": "author.get('link')" }, { "line": 465, "key": "role_label", "expression": "author.get('role_label')" }, { "line": 465, "key": "title", "expression": "author.get('title')" } ], "dictionary_entries": [ { "line": 462, "key": "source_author_id_hash", "value_expression": "provenance.content_sha256(source_author_id) if source_author_id else ''" }, { "line": 463, "key": "display_name", "value_expression": "author.get('display_name') or author.get('name') or ''" }, { "line": 464, "key": "profile_url", "value_expression": "author.get('profile_url') or author.get('link') or ''" }, { "line": 465, "key": "role_label", "value_expression": "author.get('role_label') or author.get('title') or ''" } ] }, { "symbol": "_source_snapshot", "line_start": 469, "line_end": 487, "literal_get_calls": [], "dictionary_entries": [ { "line": 472, "key": "document_id", "value_expression": "provenance.document_id(key, f'post_{post_id}', 'forum_post_html')" }, { "line": 473, "key": "document_kind", "value_expression": "'forum_post_html'" }, { "line": 474, "key": "content_sha256", "value_expression": "content_sha" }, { "line": 475, "key": "content_size_bytes", "value_expression": "len(html.encode('utf-8', errors='ignore'))" }, { "line": 476, "key": "content_type", "value_expression": "'text/html'" }, { "line": 477, "key": "url", "value_expression": "url" }, { "line": 478, "key": "retrieved_at", "value_expression": "retrieved_at" }, { "line": 479, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 480, "key": "raw_content", "value_expression": "html" }, { "line": 483, "key": "url", "value_expression": "url" }, { "line": 484, "key": "content_type", "value_expression": "'text/html'" }, { "line": 485, "key": "content_sha256", "value_expression": "content_sha" }, { "line": 486, "key": "documents", "value_expression": "[doc] if html else []" } ] }, { "symbol": "_quality_payload", "line_start": 490, "line_end": 498, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_provenance_payload", "line_start": 501, "line_end": 530, "literal_get_calls": [], "dictionary_entries": [ { "line": 510, "key": "source_id", "value_expression": "provenance.source_id(key)" }, { "line": 511, "key": "source_record_id", "value_expression": "provenance.source_record_id(key, 'post_id', post_id)" }, { "line": 512, "key": "record_id", "value_expression": "provenance.source_record_id(key, 'post_id', post_id)" }, { "line": 513, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 514, "key": "transform_run_id", "value_expression": "transform_run_id or ''" }, { "line": 515, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 516, "key": "document_ids", "value_expression": "[]" }, { "line": 520, "key": "normalized_at", "value_expression": "datetime.now(timezone.utc).isoformat()" }, { "line": 521, "key": "normalizer", "value_expression": "'aggregator.vmi_forum_transfer_schema.normalize_vmi_forum_post_record'" }, { "line": 522, "key": "warnings", "value_expression": "[]" }, { "line": 523, "key": "identity", "value_expression": "identity" }, { "line": 524, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_name': SCHEMA_NAME, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 525, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 526, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 527, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 528, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" } ] }, { "symbol": "_normalize_link", "line_start": 533, "line_end": 579, "literal_get_calls": [ { "line": 557, "key": "position", "expression": "link.get('position', index)" }, { "line": 534, "key": "url", "expression": "link.get('url')" }, { "line": 534, "key": "href", "expression": "link.get('href')" }, { "line": 537, "key": "link_text", "expression": "link.get('link_text')" }, { "line": 537, "key": "label", "expression": "link.get('label')" }, { "line": 551, "key": "rel", "expression": "link.get('rel')" }, { "line": 552, "key": "content_type_hint", "expression": "link.get('content_type_hint')" }, { "line": 558, "key": "source_selector", "expression": "link.get('source_selector')" } ], "dictionary_entries": [ { "line": 548, "key": "url", "value_expression": "url" }, { "line": 549, "key": "normalized_url", "value_expression": "url" }, { "line": 550, "key": "link_text", "value_expression": "label" }, { "line": 551, "key": "rel", "value_expression": "link.get('rel') or ''" }, { "line": 552, "key": "content_type_hint", "value_expression": "link.get('content_type_hint') or _content_type_hint(path)" }, { "line": 553, "key": "is_download_candidate", "value_expression": "bool(is_download)" }, { "line": 554, "key": "is_product_homepage_candidate", "value_expression": "is_product" }, { "line": 555, "key": "is_external_identifier_candidate", "value_expression": "bool(not internal and (not non_product_host))" }, { "line": 556, "key": "is_forum_evidence_link", "value_expression": "bool(internal)" }, { "line": 557, "key": "position", "value_expression": "link.get('position', index)" }, { "line": 558, "key": "source_selector", "value_expression": "link.get('source_selector') or {}" }, { "line": 562, "key": "eligible", "value_expression": "True" }, { "line": 563, "key": "status", "value_expression": "'candidate'" }, { "line": 564, "key": "task", "value_expression": "'vmi_product_page_analysis'" }, { "line": 565, "key": "target_entity_path", "value_expression": "'extracted_mentions'" }, { "line": 566, "key": "expected_outputs", "value_expression": "['developer_or_publisher', 'represented_organ', 'source_institution', 'engine_or_platform', 'version', 'license_or_access_hints', 'package_or_download_metadata']" }, { "line": 575, "key": "requires_review", "value_expression": "True" }, { "line": 576, "key": "fetch_policy", "value_expression": "'public_html_only'" }, { "line": 577, "key": "reason", "value_expression": "'External forum link points to a product page that may contain richer VMI evidence than the post text.'" } ] }, { "symbol": "_is_internal_forum_url", "line_start": 582, "line_end": 590, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_normalize_media", "line_start": 593, "line_end": 604, "literal_get_calls": [ { "line": 602, "key": "position", "expression": "item.get('position', index)" }, { "line": 594, "key": "url", "expression": "item.get('url')" }, { "line": 594, "key": "link", "expression": "item.get('link')" }, { "line": 595, "key": "kind", "expression": "item.get('kind')" }, { "line": 595, "key": "media_kind", "expression": "item.get('media_kind')" }, { "line": 600, "key": "file_name", "expression": "item.get('file_name')" }, { "line": 601, "key": "caption", "expression": "item.get('caption')" }, { "line": 603, "key": "status", "expression": "item.get('status')" } ], "dictionary_entries": [ { "line": 597, "key": "kind", "value_expression": "kind" }, { "line": 598, "key": "url", "value_expression": "url" }, { "line": 599, "key": "link", "value_expression": "url" }, { "line": 600, "key": "file_name", "value_expression": "item.get('file_name') or os.path.basename(urlparse(url).path)" }, { "line": 601, "key": "caption", "value_expression": "item.get('caption') or ''" }, { "line": 602, "key": "position", "value_expression": "item.get('position', index)" }, { "line": 603, "key": "status", "value_expression": "item.get('status') or 'discovered'" } ] }, { "symbol": "_with_entity_ids", "line_start": 607, "line_end": 615, "literal_get_calls": [ { "line": 613, "key": "mention_id", "expression": "value.get('mention_id')" }, { "line": 613, "key": "url", "expression": "value.get('url')" } ], "dictionary_entries": [] }, { "symbol": "_matched_terms", "line_start": 618, "line_end": 620, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_candidate_title", "line_start": 623, "line_end": 626, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_excerpt", "line_start": 629, "line_end": 631, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_clean_text", "line_start": 634, "line_end": 635, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_content_type_hint", "line_start": 638, "line_end": 645, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/musixplora/graph_builder.py", "sha256": "9f4c4bbb280a35561c48ae57cb5e445840bb22a7056372ac32b6a6ebd2a93de2", "functions": [ { "symbol": "infer_entity_family", "line_start": 61, "line_end": 81, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "resolve_source_dir", "line_start": 84, "line_end": 95, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_inverse_generation", "line_start": 520, "line_end": 525, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "is_organ_relevant", "line_start": 528, "line_end": 538, "literal_get_calls": [ { "line": 529, "key": "entity_type", "expression": "entity.get('entity_type')" }, { "line": 533, "key": "subtype", "expression": "entity.get('subtype')" }, { "line": 534, "key": "titles", "expression": "entity.get('titles', [])" }, { "line": 535, "key": "names", "expression": "entity.get('names', [])" }, { "line": 534, "key": "value", "expression": "item.get('value')" }, { "line": 535, "key": "value", "expression": "item.get('value')" } ], "dictionary_entries": [] }, { "symbol": "_empty_entity", "line_start": 541, "line_end": 563, "literal_get_calls": [], "dictionary_entries": [ { "line": 543, "key": "mxp_id", "value_expression": "str(mxp_id)" }, { "line": 544, "key": "entity_type", "value_expression": "entity_type" }, { "line": 545, "key": "entity_family", "value_expression": "family" }, { "line": 546, "key": "subtype", "value_expression": "''" }, { "line": 547, "key": "names", "value_expression": "[]" }, { "line": 548, "key": "titles", "value_expression": "[]" }, { "line": 549, "key": "occupations", "value_expression": "[]" }, { "line": 550, "key": "sectors", "value_expression": "[]" }, { "line": 551, "key": "descriptions", "value_expression": "[]" }, { "line": 552, "key": "dates", "value_expression": "[]" }, { "line": 553, "key": "properties", "value_expression": "[]" }, { "line": 554, "key": "events", "value_expression": "[]" }, { "line": 555, "key": "relations", "value_expression": "[]" }, { "line": 556, "key": "sources", "value_expression": "[]" }, { "line": 557, "key": "media", "value_expression": "[]" }, { "line": 558, "key": "literature", "value_expression": "[]" }, { "line": 559, "key": "external_references", "value_expression": "[]" }, { "line": 560, "key": "status", "value_expression": "{}" }, { "line": 561, "key": "source_tables", "value_expression": "[]" }, { "line": 562, "key": "raw_rows", "value_expression": "[]" } ] }, { "symbol": "_entity_label", "line_start": 566, "line_end": 590, "literal_get_calls": [ { "line": 569, "key": "entity_type", "expression": "entity.get('entity_type')" }, { "line": 590, "key": "mxp_id", "expression": "entity.get('mxp_id')" }, { "line": 573, "key": "names", "expression": "entity.get('names')" }, { "line": 574, "key": "value", "expression": "item.get('value')" }, { "line": 576, "key": "type", "expression": "item.get('type')" }, { "line": 588, "key": "value", "expression": "item.get('value')" }, { "line": 576, "key": "type", "expression": "item.get('type')" }, { "line": 578, "key": "position", "expression": "item.get('position')" }, { "line": 580, "key": "position", "expression": "item.get('position')" } ], "dictionary_entries": [] }, { "symbol": "_position_value", "line_start": 593, "line_end": 597, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_preferred_name_parts", "line_start": 600, "line_end": 605, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_name_is_preferred", "line_start": 608, "line_end": 611, "literal_get_calls": [ { "line": 609, "key": "raw", "expression": "item.get('raw')" }, { "line": 610, "key": "F11B", "expression": "raw.get('F11B')" }, { "line": 610, "key": "F12B", "expression": "raw.get('F12B')" }, { "line": 610, "key": "preferred", "expression": "item.get('preferred')" }, { "line": 610, "key": "is_preferred", "expression": "item.get('is_preferred')" }, { "line": 609, "key": "raw", "expression": "item.get('raw')" } ], "dictionary_entries": [] }, { "symbol": "_ensure_entity", "line_start": 614, "line_end": 622, "literal_get_calls": [], "dictionary_entries": [ { "line": 620, "key": "type", "value_expression": "family" }, { "line": 620, "key": "label", "value_expression": "family" } ] }, { "symbol": "_entity_for_attach", "line_start": 625, "line_end": 630, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_date_fields", "line_start": 633, "line_end": 651, "literal_get_calls": [], "dictionary_entries": [ { "line": 645, "key": "kind", "value_expression": "label" }, { "line": 646, "key": "display", "value_expression": "row.get(display_key) or ''" }, { "line": 647, "key": "start", "value_expression": "row.get(start_key) or ''" }, { "line": 648, "key": "end", "value_expression": "row.get(end_key) if end_key else ''" } ] }, { "symbol": "__init__", "line_start": 101, "line_end": 106, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "load", "line_start": 108, "line_end": 111, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "manifest_payload", "line_start": 113, "line_end": 115, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "build_entities", "line_start": 117, "line_end": 133, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "build_person_entities", "line_start": 135, "line_end": 153, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "build_entity", "line_start": 155, "line_end": 156, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "build_relations", "line_start": 158, "line_end": 213, "literal_get_calls": [ { "line": 161, "key": "relations_entities", "expression": "self.tables.get('relations_entities', [])" }, { "line": 186, "key": "relations_persons", "expression": "self.tables.get('relations_persons', [])" }, { "line": 166, "key": "Art", "expression": "row.get('Art')" }, { "line": 191, "key": "R", "expression": "row.get('R')" }, { "line": 192, "key": "G", "expression": "row.get('G')" }, { "line": 162, "key": "maior", "expression": "row.get('maior')" }, { "line": 163, "key": "minor", "expression": "row.get('minor')" }, { "line": 176, "key": "category", "expression": "semantic.get('category', '')" }, { "line": 177, "key": "subcategory", "expression": "semantic.get('subcategory', '')" }, { "line": 187, "key": "F41X", "expression": "row.get('F41X')" }, { "line": 188, "key": "F41Y", "expression": "row.get('F41Y')" }, { "line": 202, "key": "role_label", "expression": "relation_semantics.get('role_label', '')" }, { "line": 204, "key": "generation_label", "expression": "relation_semantics.get('generation_label', '')" }, { "line": 205, "key": "relation_category", "expression": "relation_semantics.get('relation_category', '')" }, { "line": 175, "key": "Art", "expression": "row.get('Art')" } ], "dictionary_entries": [ { "line": 170, "key": "edge_id", "value_expression": "f'mxp:edge:entity:{maior}:{minor}:{row_hash(row)[:12]}'" }, { "line": 171, "key": "source_mxp_id", "value_expression": "maior" }, { "line": 172, "key": "target_mxp_id", "value_expression": "minor" }, { "line": 173, "key": "relation_kind", "value_expression": "'entity_relation'" }, { "line": 174, "key": "role", "value_expression": "role" }, { "line": 175, "key": "role_code", "value_expression": "row.get('Art') or ''" }, { "line": 176, "key": "semantic_category", "value_expression": "semantic.get('category', '')" }, { "line": 177, "key": "semantic_subcategory", "value_expression": "semantic.get('subcategory', '')" }, { "line": 178, "key": "direction", "value_expression": "'maior_to_minor'" }, { "line": 179, "key": "inverse_role", "value_expression": "'inverse_' + str(role)" }, { "line": 180, "key": "source_table", "value_expression": "'mim_table_relation'" }, { "line": 181, "key": "source_row_hash", "value_expression": "row_hash(row)" }, { "line": 182, "key": "confidence", "value_expression": "'source_asserted'" }, { "line": 183, "key": "raw", "value_expression": "dict(row)" }, { "line": 196, "key": "edge_id", "value_expression": "f'mxp:edge:person:{source}:{target}:{row_hash(row)[:12]}'" }, { "line": 197, "key": "source_mxp_id", "value_expression": "source" }, { "line": 198, "key": "target_mxp_id", "value_expression": "target" }, { "line": 199, "key": "relation_kind", "value_expression": "'person_relation'" }, { "line": 200, "key": "role", "value_expression": "role_code or 'relationship'" }, { "line": 201, "key": "role_code", "value_expression": "role_code" }, { "line": 202, "key": "role_label", "value_expression": "relation_semantics.get('role_label', '')" }, { "line": 203, "key": "generation", "value_expression": "generation" }, { "line": 204, "key": "generation_label", "value_expression": "relation_semantics.get('generation_label', '')" }, { "line": 205, "key": "relation_category", "value_expression": "relation_semantics.get('relation_category', '')" }, { "line": 206, "key": "direction", "value_expression": "'F41X_to_F41Y'" }, { "line": 207, "key": "source_table", "value_expression": "'mim_table_beziehung'" }, { "line": 208, "key": "source_row_hash", "value_expression": "row_hash(row)" }, { "line": 209, "key": "confidence", "value_expression": "'source_asserted'" }, { "line": 210, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_person_relation_semantics", "line_start": 215, "line_end": 237, "literal_get_calls": [ { "line": 228, "key": "relations_person_semantics", "expression": "self.tables.get('relations_person_semantics', [])" }, { "line": 235, "key": "Beziehung", "expression": "reference_row.get('Beziehung')" }, { "line": 229, "key": "R", "expression": "row.get('R')" } ], "dictionary_entries": [ { "line": 234, "key": "relation_category", "value_expression": "categorize_person_relation(normalized_role)" }, { "line": 235, "key": "role_label", "value_expression": "str(reference_row.get('Beziehung') or '')" }, { "line": 236, "key": "generation_label", "value_expression": "str(reference_row.get(normalized_generation) or '')" } ] }, { "symbol": "_inverse_relation", "line_start": 239, "line_end": 267, "literal_get_calls": [ { "line": 244, "key": "target_mxp_id", "expression": "relation.get('target_mxp_id')" }, { "line": 245, "key": "source_mxp_id", "expression": "relation.get('source_mxp_id')" }, { "line": 258, "key": "relation_kind", "expression": "relation.get('relation_kind')" }, { "line": 264, "key": "generation_label", "expression": "semantics.get('generation_label', '')" }, { "line": 265, "key": "role_label", "expression": "semantics.get('role_label', '')" }, { "line": 266, "key": "relation_category", "expression": "semantics.get('relation_category', '')" }, { "line": 249, "key": "target_label", "expression": "relation.get('target_label', '')" }, { "line": 250, "key": "source_label", "expression": "relation.get('source_label', '')" }, { "line": 254, "key": "target_entity_type", "expression": "relation.get('target_entity_type', '')" }, { "line": 255, "key": "source_entity_type", "expression": "relation.get('source_entity_type', '')" }, { "line": 259, "key": "generation", "expression": "relation.get('generation')" }, { "line": 261, "key": "role_code", "expression": "relation.get('role_code')" } ], "dictionary_entries": [] }, { "symbol": "_seed_entities", "line_start": 269, "line_end": 296, "literal_get_calls": [ { "line": 292, "key": "Werkform", "expression": "row.get('Werkform')" }, { "line": 292, "key": "Medienformat", "expression": "row.get('Medienformat')" }, { "line": 292, "key": "variant_primary", "expression": "row.get('variant_primary')" }, { "line": 285, "key": "F41", "expression": "row.get('F41')" }, { "line": 285, "key": "ID", "expression": "row.get('ID')" } ], "dictionary_entries": [ { "line": 290, "key": "table", "value_expression": "spec['master']" }, { "line": 290, "key": "row_hash", "value_expression": "row_hash(row)" }, { "line": 290, "key": "row", "value_expression": "dict(row)" } ] }, { "symbol": "_person_entity_ids", "line_start": 298, "line_end": 317, "literal_get_calls": [ { "line": 301, "key": "F41", "expression": "row.get('F41')" }, { "line": 301, "key": "ID", "expression": "row.get('ID')" }, { "line": 305, "key": "F41", "expression": "row.get('F41')" }, { "line": 305, "key": "ID", "expression": "row.get('ID')" } ], "dictionary_entries": [] }, { "symbol": "_attach_person_relations", "line_start": 319, "line_end": 338, "literal_get_calls": [ { "line": 322, "key": "source_mxp_id", "expression": "relation.get('source_mxp_id')" }, { "line": 323, "key": "target_mxp_id", "expression": "relation.get('target_mxp_id')" } ], "dictionary_entries": [ { "line": 330, "key": "source_label", "value_expression": "_entity_label(source) if source else labels.get(source_id, source_id)" }, { "line": 331, "key": "target_label", "value_expression": "_entity_label(target) if target else labels.get(target_id, target_id)" }, { "line": 332, "key": "source_entity_type", "value_expression": "'person' if source else ENTITY_SPECS.get(source_family, {'type': source_family})['type']" }, { "line": 333, "key": "target_entity_type", "value_expression": "'person' if target else ENTITY_SPECS.get(target_family, {'type': target_family})['type']" }, { "line": 332, "key": "type", "value_expression": "source_family" }, { "line": 333, "key": "type", "value_expression": "target_family" } ] }, { "symbol": "_lightweight_label_index", "line_start": 340, "line_end": 354, "literal_get_calls": [ { "line": 350, "key": "F41", "expression": "row.get('F41')" } ], "dictionary_entries": [] }, { "symbol": "_attach_names_titles", "line_start": 356, "line_end": 379, "literal_get_calls": [ { "line": 357, "key": "person_surnames", "expression": "self.tables.get('person_surnames', [])" }, { "line": 363, "key": "person_forenames", "expression": "self.tables.get('person_forenames', [])" }, { "line": 358, "key": "F41", "expression": "row.get('F41')" }, { "line": 364, "key": "F41", "expression": "row.get('F41')" }, { "line": 369, "key": "F41", "expression": "row.get('F41')" }, { "line": 362, "key": "F11", "expression": "row.get('F11')" }, { "line": 362, "key": "F11A", "expression": "row.get('F11A')" }, { "line": 366, "key": "F12", "expression": "row.get('F12')" }, { "line": 366, "key": "F12A", "expression": "row.get('F12A')" }, { "line": 375, "key": "F21A", "expression": "row.get('F21A')" }, { "line": 375, "key": "F11A", "expression": "row.get('F11A')" }, { "line": 375, "key": "F32A", "expression": "row.get('F32A')" }, { "line": 375, "key": "NC", "expression": "row.get('NC')" } ], "dictionary_entries": [ { "line": 375, "key": "value", "value_expression": "value" }, { "line": 375, "key": "position", "value_expression": "row.get('F21A') or row.get('F11A') or row.get('F32A') or row.get('NC')" }, { "line": 375, "key": "raw", "value_expression": "dict(row)" }, { "line": 362, "key": "type", "value_expression": "'surname'" }, { "line": 362, "key": "value", "value_expression": "row.get('F11')" }, { "line": 362, "key": "position", "value_expression": "row.get('F11A')" }, { "line": 362, "key": "raw", "value_expression": "dict(row)" }, { "line": 366, "key": "type", "value_expression": "'forename'" }, { "line": 366, "key": "value", "value_expression": "row.get('F12')" }, { "line": 366, "key": "position", "value_expression": "row.get('F12A')" }, { "line": 366, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_person_classifications", "line_start": 381, "line_end": 417, "literal_get_calls": [ { "line": 403, "key": "sectors", "expression": "self.tables.get('sectors', [])" }, { "line": 405, "key": "F33", "expression": "row.get('F33')" }, { "line": 404, "key": "F41", "expression": "row.get('F41')" }, { "line": 389, "key": "F41", "expression": "row.get('F41')" }, { "line": 411, "key": "F33A", "expression": "row.get('F33A')" } ], "dictionary_entries": [ { "line": 409, "key": "term", "value_expression": "value" }, { "line": 410, "key": "classification_type", "value_expression": "'sector'" }, { "line": 411, "key": "position", "value_expression": "row.get('F33A') or ''" }, { "line": 412, "key": "source_table", "value_expression": "TABLE_FILES['sectors'].replace('.json', '')" }, { "line": 413, "key": "source_field", "value_expression": "'F33'" }, { "line": 414, "key": "source_row_hash", "value_expression": "row_hash(row)" }, { "line": 415, "key": "raw", "value_expression": "dict(row)" }, { "line": 394, "key": "term", "value_expression": "value" }, { "line": 395, "key": "classification_type", "value_expression": "classification_type" }, { "line": 396, "key": "position", "value_expression": "row.get(f'{value_field}A') or ''" }, { "line": 397, "key": "source_table", "value_expression": "TABLE_FILES[table].replace('.json', '')" }, { "line": 398, "key": "source_field", "value_expression": "value_field" }, { "line": 399, "key": "source_row_hash", "value_expression": "row_hash(row)" }, { "line": 400, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_descriptions", "line_start": 419, "line_end": 424, "literal_get_calls": [ { "line": 422, "key": "F41", "expression": "row.get('F41')" }, { "line": 423, "key": "Beschreibung", "expression": "row.get('Beschreibung')" }, { "line": 424, "key": "Beschreibung", "expression": "row.get('Beschreibung')" }, { "line": 424, "key": "Quelle", "expression": "row.get('Quelle')" } ], "dictionary_entries": [ { "line": 424, "key": "value", "value_expression": "row.get('Beschreibung')" }, { "line": 424, "key": "source", "value_expression": "row.get('Quelle') or ''" }, { "line": 424, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_properties", "line_start": 426, "line_end": 438, "literal_get_calls": [ { "line": 427, "key": "properties", "expression": "self.tables.get('properties', [])" }, { "line": 428, "key": "BACCAE", "expression": "row.get('BACCAE')" }, { "line": 432, "key": "Property", "expression": "row.get('Property')" }, { "line": 433, "key": "Value", "expression": "row.get('Value')" }, { "line": 434, "key": "Unit", "expression": "row.get('Unit')" }, { "line": 435, "key": "EVENTA", "expression": "row.get('EVENTA')" } ], "dictionary_entries": [ { "line": 432, "key": "property", "value_expression": "row.get('Property')" }, { "line": 433, "key": "value", "value_expression": "row.get('Value')" }, { "line": 434, "key": "unit", "value_expression": "row.get('Unit') or ''" }, { "line": 435, "key": "event_mxp_id", "value_expression": "row.get('EVENTA') or ''" }, { "line": 436, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_events", "line_start": 440, "line_end": 457, "literal_get_calls": [ { "line": 441, "key": "events", "expression": "self.tables.get('events', [])" }, { "line": 442, "key": "F41", "expression": "row.get('F41')" }, { "line": 444, "key": "Art", "expression": "row.get('Art')" }, { "line": 448, "key": "category", "expression": "semantic.get('category', '')" }, { "line": 449, "key": "subcategory", "expression": "semantic.get('subcategory', '')" }, { "line": 447, "key": "Art", "expression": "row.get('Art')" }, { "line": 450, "key": "Level", "expression": "row.get('Level')" }, { "line": 451, "key": "Ort", "expression": "row.get('Ort')" } ], "dictionary_entries": [ { "line": 447, "key": "type", "value_expression": "row.get('Art') or ''" }, { "line": 448, "key": "semantic_category", "value_expression": "semantic.get('category', '')" }, { "line": 449, "key": "semantic_subcategory", "value_expression": "semantic.get('subcategory', '')" }, { "line": 450, "key": "level", "value_expression": "row.get('Level') or ''" }, { "line": 451, "key": "place", "value_expression": "row.get('Ort') or ''" }, { "line": 452, "key": "dates", "value_expression": "_date_fields(row)" }, { "line": 453, "key": "source_table", "value_expression": "'mim_table_ereignis'" }, { "line": 454, "key": "source_row_hash", "value_expression": "row_hash(row)" }, { "line": 455, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_sources", "line_start": 459, "line_end": 463, "literal_get_calls": [ { "line": 460, "key": "sources", "expression": "self.tables.get('sources', [])" }, { "line": 461, "key": "F41", "expression": "row.get('F41')" }, { "line": 463, "key": "F51", "expression": "row.get('F51')" }, { "line": 463, "key": "F51A", "expression": "row.get('F51A')" } ], "dictionary_entries": [ { "line": 463, "key": "reference", "value_expression": "row.get('F51') or ''" }, { "line": 463, "key": "position", "value_expression": "row.get('F51A') or ''" }, { "line": 463, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_media", "line_start": 465, "line_end": 478, "literal_get_calls": [ { "line": 466, "key": "images", "expression": "self.tables.get('images', [])" }, { "line": 467, "key": "F41", "expression": "row.get('F41')" }, { "line": 472, "key": "Link", "expression": "row.get('Link')" }, { "line": 473, "key": "Credits", "expression": "row.get('Credits')" }, { "line": 474, "key": "Source", "expression": "row.get('Source')" }, { "line": 475, "key": "last_check", "expression": "row.get('last_check')" } ], "dictionary_entries": [ { "line": 471, "key": "kind", "value_expression": "'image'" }, { "line": 472, "key": "url", "value_expression": "row.get('Link') or ''" }, { "line": 473, "key": "credits", "value_expression": "row.get('Credits') or ''" }, { "line": 474, "key": "source", "value_expression": "row.get('Source') or ''" }, { "line": 475, "key": "last_check", "value_expression": "row.get('last_check') or ''" }, { "line": 476, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_externals", "line_start": 480, "line_end": 491, "literal_get_calls": [ { "line": 481, "key": "externals", "expression": "self.tables.get('externals', [])" }, { "line": 482, "key": "F41", "expression": "row.get('F41')" }, { "line": 486, "key": "Referenz", "expression": "row.get('Referenz')" }, { "line": 487, "key": "F45", "expression": "row.get('F45')" }, { "line": 488, "key": "Status", "expression": "row.get('Status')" } ], "dictionary_entries": [ { "line": 486, "key": "authority", "value_expression": "row.get('Referenz') or ''" }, { "line": 487, "key": "identifier", "value_expression": "row.get('F45') or ''" }, { "line": 488, "key": "status", "value_expression": "row.get('Status') or ''" }, { "line": 489, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_literature", "line_start": 493, "line_end": 497, "literal_get_calls": [ { "line": 494, "key": "literature", "expression": "self.tables.get('literature', [])" }, { "line": 495, "key": "F41", "expression": "row.get('F41')" }, { "line": 497, "key": "Bibtex", "expression": "row.get('Bibtex')" } ], "dictionary_entries": [ { "line": 497, "key": "bibtex", "value_expression": "row.get('Bibtex') or ''" }, { "line": 497, "key": "raw", "value_expression": "dict(row)" } ] }, { "symbol": "_attach_relations", "line_start": 499, "line_end": 513, "literal_get_calls": [ { "line": 501, "key": "source_mxp_id", "expression": "relation.get('source_mxp_id')" }, { "line": 502, "key": "target_mxp_id", "expression": "relation.get('target_mxp_id')" }, { "line": 507, "key": "entity_type", "expression": "source.get('entity_type')" }, { "line": 508, "key": "entity_type", "expression": "target.get('entity_type')" } ], "dictionary_entries": [ { "line": 505, "key": "source_label", "value_expression": "_entity_label(source) if source else ''" }, { "line": 506, "key": "target_label", "value_expression": "_entity_label(target) if target else ''" }, { "line": 507, "key": "source_entity_type", "value_expression": "source.get('entity_type') if source else ''" }, { "line": 508, "key": "target_entity_type", "value_expression": "target.get('entity_type') if target else ''" } ] }, { "symbol": "_ensure_loaded", "line_start": 515, "line_end": 517, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/musixplora/transfer_schema.py", "sha256": "634d2902308582cbfdfb40d8dd375652c36b4d14a01e8b7f523c4276313bba03", "functions": [ { "symbol": "normalize_musixplora_entity_record", "line_start": 23, "line_end": 105, "literal_get_calls": [ { "line": 35, "key": "mxp_id", "expression": "payload.get('mxp_id')" }, { "line": 53, "key": "relations", "expression": "payload.get('relations')" }, { "line": 54, "key": "events", "expression": "payload.get('events')" }, { "line": 55, "key": "occupations", "expression": "payload.get('occupations')" }, { "line": 56, "key": "sectors", "expression": "payload.get('sectors')" }, { "line": 58, "key": "media", "expression": "payload.get('media')" }, { "line": 59, "key": "literature", "expression": "payload.get('literature')" }, { "line": 60, "key": "external_references", "expression": "payload.get('external_references')" }, { "line": 91, "key": "source_tables", "expression": "payload.get('source_tables', [])" }, { "line": 92, "key": "raw_rows", "expression": "payload.get('raw_rows', [])" }, { "line": 94, "key": "sources", "expression": "payload.get('sources', [])" }, { "line": 95, "key": "status", "expression": "payload.get('status', {})" } ], "dictionary_entries": [ { "line": 39, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 40, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 41, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 42, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 43, "key": "record_type", "value_expression": "RECORD_TYPE" }, { "line": 44, "key": "source", "value_expression": "{'key': SOURCE_KEY, 'name': 'musiXplora', 'base_url': BASE_URL, 'type': 'file_dump', 'retrieved_at': retrieved_at or datetime.now(timezone.utc).isoformat()}" }, { "line": 51, "key": "identifiers", "value_expression": "_identifiers(payload)" }, { "line": 52, "key": "entity", "value_expression": "_entity_payload(payload)" }, { "line": 53, "key": "relations", "value_expression": "_as_list(payload.get('relations'))" }, { "line": 54, "key": "events", "value_expression": "_as_list(payload.get('events'))" }, { "line": 55, "key": "occupations", "value_expression": "_as_list(payload.get('occupations'))" }, { "line": 56, "key": "sectors", "value_expression": "_as_list(payload.get('sectors'))" }, { "line": 57, "key": "source_term_occurrences", "value_expression": "_source_term_occurrences(payload, mxp_id=mxp_id)" }, { "line": 58, "key": "media", "value_expression": "_as_list(payload.get('media'))" }, { "line": 59, "key": "literature", "value_expression": "_as_list(payload.get('literature'))" }, { "line": 60, "key": "external_references", "value_expression": "_as_list(payload.get('external_references'))" }, { "line": 61, "key": "source_snapshot", "value_expression": "{'url': f'{BASE_URL}/mxp/{mxp_id}' if mxp_id else BASE_URL, 'content_type': 'application/json', 'raw_content': raw_content, 'content_sha256': provenance.content_sha256(raw_content), 'documents': []}" }, { "line": 68, "key": "provenance", "value_expression": "{'normalized_at': normalized_at or datetime.now(timezone.utc).isoformat(), 'normalizer': 'aggregator.musixplora.transfer_schema.normalize_musixplora_entity_record', 'warnings': [], 'identity': {'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': provenance.source_record_id(SOURCE_KEY, 'mxp_id', identity_value), 'record_id': provenance.source_record_id(SOURCE_KEY, 'mxp_id', identity_value), 'crawl_run_id': crawl_run_id or '', 'transform_run_id': transform_run_id or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': []}, 'schema': {'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_name': SCHEMA_NAME, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}}" }, { "line": 88, "key": "quality", "value_expression": "{'schema_valid': None, 'completeness': {}}" }, { "line": 89, "key": "extensions", "value_expression": "{'musixplora': {'source_tables': payload.get('source_tables', []), 'raw_rows': payload.get('raw_rows', []), 'dump_manifest': list(manifest or []), 'sources': payload.get('sources', []), 'status': payload.get('status', {})}}" }, { "line": 45, "key": "key", "value_expression": "SOURCE_KEY" }, { "line": 46, "key": "name", "value_expression": "'musiXplora'" }, { "line": 47, "key": "base_url", "value_expression": "BASE_URL" }, { "line": 48, "key": "type", "value_expression": "'file_dump'" }, { "line": 49, "key": "retrieved_at", "value_expression": "retrieved_at or datetime.now(timezone.utc).isoformat()" }, { "line": 62, "key": "url", "value_expression": "f'{BASE_URL}/mxp/{mxp_id}' if mxp_id else BASE_URL" }, { "line": 63, "key": "content_type", "value_expression": "'application/json'" }, { "line": 64, "key": "raw_content", "value_expression": "raw_content" }, { "line": 65, "key": "content_sha256", "value_expression": "provenance.content_sha256(raw_content)" }, { "line": 66, "key": "documents", "value_expression": "[]" }, { "line": 69, "key": "normalized_at", "value_expression": "normalized_at or datetime.now(timezone.utc).isoformat()" }, { "line": 70, "key": "normalizer", "value_expression": "'aggregator.musixplora.transfer_schema.normalize_musixplora_entity_record'" }, { "line": 71, "key": "warnings", "value_expression": "[]" }, { "line": 72, "key": "identity", "value_expression": "{'source_id': provenance.source_id(SOURCE_KEY), 'source_record_id': provenance.source_record_id(SOURCE_KEY, 'mxp_id', identity_value), 'record_id': provenance.source_record_id(SOURCE_KEY, 'mxp_id', identity_value), 'crawl_run_id': crawl_run_id or '', 'transform_run_id': transform_run_id or '', 'artifact_ids': list(artifact_ids or []), 'document_ids': []}" }, { "line": 81, "key": "schema", "value_expression": "{'schema_key': SCHEMA_KEY, 'schema_version': SCHEMA_VERSION, 'schema_name': SCHEMA_NAME, 'schema_artifact_uri': SCHEMA_ARTIFACT_URI}" }, { "line": 88, "key": "schema_valid", "value_expression": "None" }, { "line": 88, "key": "completeness", "value_expression": "{}" }, { "line": 90, "key": "musixplora", "value_expression": "{'source_tables': payload.get('source_tables', []), 'raw_rows': payload.get('raw_rows', []), 'dump_manifest': list(manifest or []), 'sources': payload.get('sources', []), 'status': payload.get('status', {})}" }, { "line": 73, "key": "source_id", "value_expression": "provenance.source_id(SOURCE_KEY)" }, { "line": 74, "key": "source_record_id", "value_expression": "provenance.source_record_id(SOURCE_KEY, 'mxp_id', identity_value)" }, { "line": 75, "key": "record_id", "value_expression": "provenance.source_record_id(SOURCE_KEY, 'mxp_id', identity_value)" }, { "line": 76, "key": "crawl_run_id", "value_expression": "crawl_run_id or ''" }, { "line": 77, "key": "transform_run_id", "value_expression": "transform_run_id or ''" }, { "line": 78, "key": "artifact_ids", "value_expression": "list(artifact_ids or [])" }, { "line": 79, "key": "document_ids", "value_expression": "[]" }, { "line": 82, "key": "schema_key", "value_expression": "SCHEMA_KEY" }, { "line": 83, "key": "schema_version", "value_expression": "SCHEMA_VERSION" }, { "line": 84, "key": "schema_name", "value_expression": "SCHEMA_NAME" }, { "line": 85, "key": "schema_artifact_uri", "value_expression": "SCHEMA_ARTIFACT_URI" }, { "line": 91, "key": "source_tables", "value_expression": "payload.get('source_tables', [])" }, { "line": 92, "key": "raw_rows", "value_expression": "payload.get('raw_rows', [])" }, { "line": 93, "key": "dump_manifest", "value_expression": "list(manifest or [])" }, { "line": 94, "key": "sources", "value_expression": "payload.get('sources', [])" }, { "line": 95, "key": "status", "value_expression": "payload.get('status', {})" } ] }, { "symbol": "_identifiers", "line_start": 108, "line_end": 125, "literal_get_calls": [ { "line": 111, "key": "external_references", "expression": "entity.get('external_references')" }, { "line": 114, "key": "identifier", "expression": "ref.get('identifier')" }, { "line": 109, "key": "mxp_id", "expression": "entity.get('mxp_id')" }, { "line": 118, "key": "authority", "expression": "ref.get('authority')" }, { "line": 122, "key": "status", "expression": "ref.get('status')" } ], "dictionary_entries": [ { "line": 110, "key": "scheme", "value_expression": "'mxp_id'" }, { "line": 110, "key": "value", "value_expression": "mxp_id" }, { "line": 110, "key": "role", "value_expression": "'primary'" }, { "line": 110, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 118, "key": "scheme", "value_expression": "ref.get('authority') or 'external'" }, { "line": 119, "key": "value", "value_expression": "str(value)" }, { "line": 120, "key": "role", "value_expression": "'authority'" }, { "line": 121, "key": "source", "value_expression": "SOURCE_KEY" }, { "line": 122, "key": "status", "value_expression": "ref.get('status') or ''" } ] }, { "symbol": "_entity_payload", "line_start": 128, "line_end": 142, "literal_get_calls": [ { "line": 131, "key": "entity_type", "expression": "entity.get('entity_type')" }, { "line": 132, "key": "entity_family", "expression": "entity.get('entity_family')" }, { "line": 133, "key": "subtype", "expression": "entity.get('subtype')" }, { "line": 134, "key": "names", "expression": "entity.get('names')" }, { "line": 135, "key": "titles", "expression": "entity.get('titles')" }, { "line": 136, "key": "descriptions", "expression": "entity.get('descriptions')" }, { "line": 137, "key": "dates", "expression": "entity.get('dates')" }, { "line": 138, "key": "occupations", "expression": "entity.get('occupations')" }, { "line": 139, "key": "sectors", "expression": "entity.get('sectors')" }, { "line": 140, "key": "properties", "expression": "entity.get('properties')" }, { "line": 130, "key": "mxp_id", "expression": "entity.get('mxp_id')" }, { "line": 141, "key": "status", "expression": "entity.get('status')" } ], "dictionary_entries": [ { "line": 130, "key": "mxp_id", "value_expression": "str(entity.get('mxp_id') or '')" }, { "line": 131, "key": "entity_type", "value_expression": "entity.get('entity_type') or ''" }, { "line": 132, "key": "entity_family", "value_expression": "entity.get('entity_family') or ''" }, { "line": 133, "key": "subtype", "value_expression": "entity.get('subtype') or ''" }, { "line": 134, "key": "names", "value_expression": "_as_list(entity.get('names'))" }, { "line": 135, "key": "titles", "value_expression": "_as_list(entity.get('titles'))" }, { "line": 136, "key": "descriptions", "value_expression": "_as_list(entity.get('descriptions'))" }, { "line": 137, "key": "dates", "value_expression": "_as_list(entity.get('dates'))" }, { "line": 138, "key": "occupations", "value_expression": "_as_list(entity.get('occupations'))" }, { "line": 139, "key": "sectors", "value_expression": "_as_list(entity.get('sectors'))" }, { "line": 140, "key": "properties", "value_expression": "_as_list(entity.get('properties'))" }, { "line": 141, "key": "status", "value_expression": "dict(entity.get('status') or {})" } ] }, { "symbol": "_apply_runtime_identity", "line_start": 145, "line_end": 149, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_apply_entity_ids", "line_start": 152, "line_end": 166, "literal_get_calls": [ { "line": 153, "key": "mxp_id", "expression": "record['entity'].get('mxp_id')" }, { "line": 160, "key": "edge_id", "expression": "item.get('edge_id')" }, { "line": 161, "key": "occurrence_key", "expression": "item.get('occurrence_key')" }, { "line": 162, "key": "url", "expression": "item.get('url')" }, { "line": 163, "key": "identifier", "expression": "item.get('identifier')" } ], "dictionary_entries": [] }, { "symbol": "_apply_document_identity", "line_start": 169, "line_end": 184, "literal_get_calls": [ { "line": 170, "key": "mxp_id", "expression": "record['entity'].get('mxp_id')" }, { "line": 178, "key": "url", "expression": "snapshot.get('url', '')" }, { "line": 179, "key": "retrieved_at", "expression": "record.get('source', {}).get('retrieved_at', '')" }, { "line": 180, "key": "crawl_run_id", "expression": "record['provenance']['identity'].get('crawl_run_id', '')" }, { "line": 181, "key": "source_tables", "expression": "payload.get('source_tables', [])" }, { "line": 179, "key": "source", "expression": "record.get('source', {})" } ], "dictionary_entries": [ { "line": 174, "key": "document_id", "value_expression": "provenance.document_id(SOURCE_KEY, identity_value, 'source_entity_json')" }, { "line": 175, "key": "document_kind", "value_expression": "'raw_json'" }, { "line": 176, "key": "content_sha256", "value_expression": "provenance.canonical_json_hash(payload)" }, { "line": 177, "key": "content_type", "value_expression": "'application/json'" }, { "line": 178, "key": "url", "value_expression": "snapshot.get('url', '')" }, { "line": 179, "key": "retrieved_at", "value_expression": "record.get('source', {}).get('retrieved_at', '')" }, { "line": 180, "key": "crawl_run_id", "value_expression": "record['provenance']['identity'].get('crawl_run_id', '')" }, { "line": 181, "key": "metadata", "value_expression": "{'source_tables': payload.get('source_tables', []), 'source_document_role': 'source_entity_json'}" }, { "line": 181, "key": "source_tables", "value_expression": "payload.get('source_tables', [])" }, { "line": 181, "key": "source_document_role", "value_expression": "'source_entity_json'" } ] }, { "symbol": "_identity_value", "line_start": 187, "line_end": 194, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_as_list", "line_start": 197, "line_end": 204, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_source_term_occurrences", "line_start": 207, "line_end": 261, "literal_get_calls": [ { "line": 210, "key": "occupations", "expression": "entity.get('occupations')" }, { "line": 210, "key": "sectors", "expression": "entity.get('sectors')" }, { "line": 214, "key": "term", "expression": "item.get('term')" }, { "line": 217, "key": "source_table", "expression": "item.get('source_table')" }, { "line": 218, "key": "source_field", "expression": "item.get('source_field')" }, { "line": 220, "key": "source_row_hash", "expression": "item.get('source_row_hash')" }, { "line": 219, "key": "classification_type", "expression": "item.get('classification_type')" }, { "line": 235, "key": "entity_family", "expression": "entity.get('entity_family')" }, { "line": 236, "key": "subtype", "expression": "entity.get('subtype')" }, { "line": 238, "key": "position", "expression": "item.get('position')" }, { "line": 251, "key": "entity_type", "expression": "entity.get('entity_type')" }, { "line": 252, "key": "language_code", "expression": "item.get('language_code')" }, { "line": 228, "key": "position", "expression": "item.get('position')" } ], "dictionary_entries": [ { "line": 234, "key": "source_path", "value_expression": "source_path" }, { "line": 235, "key": "entity_family", "value_expression": "entity.get('entity_family') or ''" }, { "line": 236, "key": "subtype", "value_expression": "entity.get('subtype') or ''" }, { "line": 237, "key": "classification_type", "value_expression": "source_category" }, { "line": 238, "key": "position", "value_expression": "item.get('position') or ''" }, { "line": 239, "key": "source_row_hash", "value_expression": "source_row_hash" }, { "line": 240, "key": "controlled_vocabulary", "value_expression": "source_category != 'historical_occupation_or_title'" }, { "line": 244, "key": "occurrence_key", "value_expression": "occurrence_key" }, { "line": 245, "key": "source_key", "value_expression": "SOURCE_KEY" }, { "line": 246, "key": "source_record_key", "value_expression": "source_record_key" }, { "line": 247, "key": "source_table", "value_expression": "source_table" }, { "line": 248, "key": "source_field", "value_expression": "source_field" }, { "line": 249, "key": "source_category", "value_expression": "source_category" }, { "line": 250, "key": "record_type", "value_expression": "RECORD_TYPE" }, { "line": 251, "key": "entity_type", "value_expression": "entity.get('entity_type') or ''" }, { "line": 252, "key": "language_code", "value_expression": "item.get('language_code') or 'de'" }, { "line": 253, "key": "raw_term", "value_expression": "raw_term" }, { "line": 254, "key": "normalized_term", "value_expression": "raw_term" }, { "line": 255, "key": "term_class", "value_expression": "source_category" }, { "line": 256, "key": "observed_value", "value_expression": "raw_term" }, { "line": 257, "key": "source_row_hash", "value_expression": "source_row_hash" }, { "line": 258, "key": "context_json", "value_expression": "context_json" } ] }, { "symbol": "_clean_term", "line_start": 264, "line_end": 265, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/musixplora/organ_projection.py", "sha256": "25623ec347a56ffb98f8335498f1b772e96b87bab469c3f4789de7bcc2410eed", "functions": [ { "symbol": "project_entity_to_pipe_organ_record", "line_start": 12, "line_end": 90, "literal_get_calls": [ { "line": 22, "key": "value", "expression": "item.get('value')" }, { "line": 21, "key": "mxp_id", "expression": "entity.get('mxp_id')" }, { "line": 22, "key": "titles", "expression": "entity.get('titles', [])" }, { "line": 24, "key": "value", "expression": "item.get('value')" }, { "line": 25, "key": "descriptions", "expression": "entity.get('descriptions', [])" }, { "line": 36, "key": "events", "expression": "entity.get('events', [])" }, { "line": 55, "key": "bibtex", "expression": "item.get('bibtex')" }, { "line": 55, "key": "bibtex", "expression": "item.get('bibtex')" }, { "line": 56, "key": "literature", "expression": "entity.get('literature', [])" }, { "line": 22, "key": "value", "expression": "item.get('value')" }, { "line": 26, "key": "value", "expression": "item.get('value')" }, { "line": 30, "key": "type", "expression": "item.get('type')" }, { "line": 31, "key": "dates", "expression": "item.get('dates')" }, { "line": 42, "key": "url", "expression": "item.get('url')" }, { "line": 43, "key": "url", "expression": "item.get('url')" }, { "line": 44, "key": "credits", "expression": "item.get('credits')" }, { "line": 47, "key": "media", "expression": "entity.get('media', [])" }, { "line": 57, "key": "bibtex", "expression": "item.get('bibtex')" }, { "line": 69, "key": "entity_type", "expression": "entity.get('entity_type')" }, { "line": 70, "key": "entity_family", "expression": "entity.get('entity_family')" }, { "line": 72, "key": "properties", "expression": "entity.get('properties', [])" }, { "line": 32, "key": "place", "expression": "item.get('place')" }, { "line": 45, "key": "source", "expression": "item.get('source')" }, { "line": 48, "key": "url", "expression": "item.get('url')" }, { "line": 74, "key": "edge_id", "expression": "item.get('edge_id')" }, { "line": 48, "key": "kind", "expression": "item.get('kind')" }, { "line": 74, "key": "relations", "expression": "entity.get('relations', [])" }, { "line": 74, "key": "edge_id", "expression": "item.get('edge_id')" } ], "dictionary_entries": [ { "line": 40, "key": "images", "value_expression": "[{'url': item.get('url'), 'link': item.get('url'), 'credits': item.get('credits'), 'source': item.get('source') or 'musixplora'} for item in entity.get('media', []) if isinstance(item, Mapping) and item.get('kind') == 'image' and item.get('url')]" }, { "line": 50, "key": "audio", "value_expression": "[]" }, { "line": 51, "key": "videos", "value_expression": "[]" }, { "line": 52, "key": "documents", "value_expression": "[]" }, { "line": 60, "key": "musixplora_id", "value_expression": "mxp_id" }, { "line": 61, "key": "descriptions", "value_expression": "descriptions" }, { "line": 62, "key": "events", "value_expression": "events" }, { "line": 63, "key": "media", "value_expression": "media" }, { "line": 64, "key": "literature", "value_expression": "literature" }, { "line": 65, "key": "links", "value_expression": "[{'type': 'source', 'link': f'https://musixplora.de/mxp/{mxp_id}', 'host': 'musixplora.de'}]" }, { "line": 66, "key": "extensions", "value_expression": "{'musixplora': {'source_entity_id': mxp_id, 'source_entity_type': entity.get('entity_type'), 'source_entity_family': entity.get('entity_family'), 'titles': titles, 'properties': entity.get('properties', []), 'relation_edge_ids': [item.get('edge_id') for item in entity.get('relations', []) if isinstance(item, Mapping) and item.get('edge_id')]}}" }, { "line": 24, "key": "type", "value_expression": "'source_description'" }, { "line": 24, "key": "description_text", "value_expression": "item.get('value')" }, { "line": 30, "key": "type", "value_expression": "item.get('type') or 'musiXplora event'" }, { "line": 31, "key": "dates", "value_expression": "item.get('dates') or []" }, { "line": 32, "key": "location", "value_expression": "{'raw_str': item.get('place') or ''}" }, { "line": 33, "key": "source", "value_expression": "'musixplora'" }, { "line": 34, "key": "extensions", "value_expression": "{'musixplora': item}" }, { "line": 55, "key": "type", "value_expression": "'bibtex'" }, { "line": 55, "key": "bibtex", "value_expression": "item.get('bibtex')" }, { "line": 55, "key": "raw_str", "value_expression": "item.get('bibtex')" }, { "line": 67, "key": "musixplora", "value_expression": "{'source_entity_id': mxp_id, 'source_entity_type': entity.get('entity_type'), 'source_entity_family': entity.get('entity_family'), 'titles': titles, 'properties': entity.get('properties', []), 'relation_edge_ids': [item.get('edge_id') for item in entity.get('relations', []) if isinstance(item, Mapping) and item.get('edge_id')]}" }, { "line": 32, "key": "raw_str", "value_expression": "item.get('place') or ''" }, { "line": 34, "key": "musixplora", "value_expression": "item" }, { "line": 42, "key": "url", "value_expression": "item.get('url')" }, { "line": 43, "key": "link", "value_expression": "item.get('url')" }, { "line": 44, "key": "credits", "value_expression": "item.get('credits')" }, { "line": 45, "key": "source", "value_expression": "item.get('source') or 'musixplora'" }, { "line": 65, "key": "type", "value_expression": "'source'" }, { "line": 65, "key": "link", "value_expression": "f'https://musixplora.de/mxp/{mxp_id}'" }, { "line": 65, "key": "host", "value_expression": "'musixplora.de'" }, { "line": 68, "key": "source_entity_id", "value_expression": "mxp_id" }, { "line": 69, "key": "source_entity_type", "value_expression": "entity.get('entity_type')" }, { "line": 70, "key": "source_entity_family", "value_expression": "entity.get('entity_family')" }, { "line": 71, "key": "titles", "value_expression": "titles" }, { "line": 72, "key": "properties", "value_expression": "entity.get('properties', [])" }, { "line": 73, "key": "relation_edge_ids", "value_expression": "[item.get('edge_id') for item in entity.get('relations', []) if isinstance(item, Mapping) and item.get('edge_id')]" }, { "line": 80, "key": "type", "value_expression": "'title'" }, { "line": 80, "key": "description_text", "value_expression": "titles[0]" } ] } ] }, { "path": "aggregator/ohs_stoplist_text.py", "sha256": "8e4a28f9e605e7912207d26b587afc96b14a68ea48c5068fbd122f62366f1ae3", "functions": [ { "symbol": "_clean", "line_start": 136, "line_end": 137, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_roman_value", "line_start": 140, "line_end": 152, "literal_get_calls": [], "dictionary_entries": [ { "line": 145, "key": "I", "value_expression": "1" }, { "line": 145, "key": "V", "value_expression": "5" }, { "line": 145, "key": "X", "value_expression": "10" } ] }, { "symbol": "accessory_quantity", "line_start": 155, "line_end": 160, "literal_get_calls": [], "dictionary_entries": [ { "line": 160, "key": "value", "value_expression": "int(match['count'])" }, { "line": 160, "key": "unit", "value_expression": "match['unit']" }, { "line": 160, "key": "start", "value_expression": "offset + match.start('count')" }, { "line": 160, "key": "end", "value_expression": "offset + match.end('count')" } ] }, { "symbol": "_canonical_division", "line_start": 163, "line_end": 190, "literal_get_calls": [], "dictionary_entries": [ { "line": 166, "key": "pedale", "value_expression": "'Pedal'" }, { "line": 167, "key": "pédale", "value_expression": "'Pedal'" }, { "line": 168, "key": "pedalier", "value_expression": "'Pedal'" }, { "line": 169, "key": "pedal organ", "value_expression": "'Pedal'" }, { "line": 170, "key": "great organ", "value_expression": "'Great'" }, { "line": 171, "key": "swell organ", "value_expression": "'Swell'" }, { "line": 172, "key": "choir organ", "value_expression": "'Choir'" }, { "line": 173, "key": "solo organ", "value_expression": "'Solo'" }, { "line": 174, "key": "echo organ", "value_expression": "'Echo'" }, { "line": 175, "key": "gr", "value_expression": "'Great'" }, { "line": 176, "key": "gt", "value_expression": "'Great'" }, { "line": 177, "key": "sw", "value_expression": "'Swell'" }, { "line": 178, "key": "ch", "value_expression": "'Choir'" }, { "line": 179, "key": "ped", "value_expression": "'Pedal'" }, { "line": 180, "key": "pd", "value_expression": "'Pedal'" } ] }, { "symbol": "_has_stop_row_signal", "line_start": 193, "line_end": 205, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_has_strong_stop_signal", "line_start": 208, "line_end": 234, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_inline_layout_segments", "line_start": 237, "line_end": 248, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_header_matches", "line_start": 251, "line_end": 278, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_whole_line_division_heading", "line_start": 281, "line_end": 323, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_segments", "line_start": 326, "line_end": 344, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_source_record", "line_start": 347, "line_end": 519, "literal_get_calls": [], "dictionary_entries": [ { "line": 500, "key": "name", "value_expression": "name" }, { "line": 501, "key": "division", "value_expression": "division" }, { "line": 502, "key": "length", "value_expression": "pitch" }, { "line": 503, "key": "rank_count", "value_expression": "rank_count" }, { "line": 504, "key": "is_reed", "value_expression": "bool(_REED_RE.search(name))" }, { "line": 505, "key": "raw_str", "value_expression": "_clean(raw)" }, { "line": 506, "key": "source_line", "value_expression": "line_number" }, { "line": 507, "key": "source_stoplist_id", "value_expression": "stoplist_id" }, { "line": 367, "key": "from", "value_expression": "_clean(coupler.group('from'))" }, { "line": 368, "key": "to", "value_expression": "_clean(coupler.group('to'))" }, { "line": 369, "key": "length", "value_expression": "_clean(coupler.group('length'))" }, { "line": 370, "key": "raw_str", "value_expression": "_clean(raw)" }, { "line": 371, "key": "source_line", "value_expression": "line_number" }, { "line": 372, "key": "source_stoplist_id", "value_expression": "stoplist_id" }, { "line": 385, "key": "name", "value_expression": "_clean(text)" }, { "line": 386, "key": "division", "value_expression": "division or None" }, { "line": 387, "key": "raw_str", "value_expression": "_clean(raw)" }, { "line": 388, "key": "source_line", "value_expression": "line_number" }, { "line": 389, "key": "source_stoplist_id", "value_expression": "stoplist_id" }, { "line": 390, "key": "unit_count", "value_expression": "accessory_count['value']" }, { "line": 390, "key": "unit_count_evidence", "value_expression": "accessory_count" } ] }, { "symbol": "parse_ohs_text_stoplist", "line_start": 522, "line_end": 711, "literal_get_calls": [ { "line": 678, "key": "name", "expression": "record.get('name')" } ], "dictionary_entries": [ { "line": 695, "key": "contractVersion", "value_expression": "PARSER_CONTRACT" }, { "line": 696, "key": "parserVersion", "value_expression": "PARSER_VERSION" }, { "line": 697, "key": "filename", "value_expression": "filename" }, { "line": 698, "key": "contentSha256", "value_expression": "hashlib.sha256(normalized.encode('utf-8')).hexdigest()" }, { "line": 699, "key": "lineCount", "value_expression": "len(normalized.split('\\n'))" }, { "line": 700, "key": "divisionCount", "value_expression": "len(divisions)" }, { "line": 701, "key": "stopCount", "value_expression": "len(stops)" }, { "line": 702, "key": "couplerCount", "value_expression": "len(couplers)" }, { "line": 703, "key": "accessoryCount", "value_expression": "len(accessories)" }, { "line": 704, "key": "unresolvedLineCount", "value_expression": "len(unresolved)" }, { "line": 705, "key": "divisions", "value_expression": "divisions" }, { "line": 706, "key": "stops", "value_expression": "stops" }, { "line": 707, "key": "couplers", "value_expression": "couplers" }, { "line": 708, "key": "accessories", "value_expression": "accessories" }, { "line": 709, "key": "unresolvedLines", "value_expression": "unresolved" }, { "line": 710, "key": "rawText", "value_expression": "normalized" }, { "line": 632, "key": "line", "value_expression": "line_number" }, { "line": 632, "key": "text", "value_expression": "_clean(line)" }, { "line": 629, "key": "stop", "value_expression": "stops" }, { "line": 629, "key": "coupler", "value_expression": "couplers" }, { "line": 629, "key": "accessory", "value_expression": "accessories" }, { "line": 567, "key": "stop", "value_expression": "stops" }, { "line": 567, "key": "coupler", "value_expression": "couplers" }, { "line": 567, "key": "accessory", "value_expression": "accessories" } ] } ] }, { "path": "aggregator/ohs_event_participants.py", "sha256": "af6da10cbf129e1fa0819132241fdc19e6428e48a3e0cd59faf60beeda737dce", "functions": [ { "symbol": "clean", "line_start": 16, "line_end": 17, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "participation", "line_start": 20, "line_end": 58, "literal_get_calls": [ { "line": 43, "key": "id", "expression": "builder.get('id')" }, { "line": 40, "key": "name", "expression": "builder.get('name')" }, { "line": 46, "key": "name", "expression": "builder.get('name')" } ], "dictionary_entries": [ { "line": 49, "key": "contract", "value_expression": "CONTRACT" }, { "line": 49, "key": "eventType", "value_expression": "event_type" }, { "line": 49, "key": "sourceRole", "value_expression": "role" }, { "line": 50, "key": "namePath", "value_expression": "name_path" }, { "line": 50, "key": "builderPath", "value_expression": "f'{root}{field}'" }, { "line": 51, "key": "datePaths", "value_expression": "[f'{root}{key}' for key in (prefix, year_field, suffix)]" }, { "line": 52, "key": "dateFields", "value_expression": "{key: instrument.get(key) for key in (prefix, year_field, suffix)}" }, { "line": 53, "key": "dateExpression", "value_expression": "expression" }, { "line": 53, "key": "unknownBuilder", "value_expression": "unknown" }, { "line": 54, "key": "canonicalIdentityEstablished", "value_expression": "False" }, { "line": 56, "key": "name", "value_expression": "name" }, { "line": 56, "key": "raw_str", "value_expression": "name" }, { "line": 56, "key": "activities", "value_expression": "[role]" }, { "line": 56, "key": "roles", "value_expression": "[role]" }, { "line": 57, "key": "dates", "value_expression": "[expression]" }, { "line": 57, "key": "source_id", "value_expression": "native_id" }, { "line": 58, "key": "native_event_evidence", "value_expression": "evidence" } ] } ] }, { "path": "aggregator/schmidt_history.py", "sha256": "966a29b46a8f81b967304662311fa06cdfbf896c607572fe9f8e915babccb5fd", "functions": [ { "symbol": "span", "line_start": 24, "line_end": 25, "literal_get_calls": [], "dictionary_entries": [ { "line": 25, "key": "start", "value_expression": "start" }, { "line": 25, "key": "end", "value_expression": "end" }, { "line": 25, "key": "wording", "value_expression": "text[start:end]" } ] }, { "symbol": "mention", "line_start": 28, "line_end": 91, "literal_get_calls": [], "dictionary_entries": [ { "line": 56, "key": "reason", "value_expression": "'unsupported_actor_clause'" }, { "line": 59, "key": "reason", "value_expression": "'insufficient_actor_context'" }, { "line": 61, "key": "reason", "value_expression": "'ambiguous_compound_actor_clause'" }, { "line": 86, "key": "name", "value_expression": "name" }, { "line": 86, "key": "activities", "value_expression": "[role]" }, { "line": 86, "key": "role", "value_expression": "role" }, { "line": 87, "key": "actor_type", "value_expression": "kind" }, { "line": 87, "key": "place_context", "value_expression": "place" }, { "line": 88, "key": "evidence_spans", "value_expression": "evidence" }, { "line": 89, "key": "name_derivation", "value_expression": "'shared_surname' if shared else 'literal'" }, { "line": 90, "key": "resolution_state", "value_expression": "'unresolved_source_mention'" }, { "line": 49, "key": "reason", "value_expression": "'unbalanced_name_context'" }, { "line": 66, "key": "reason", "value_expression": "'ambiguous_coordinated_name'" }, { "line": 69, "key": "reason", "value_expression": "'ambiguous_shared_surname'" } ] }, { "symbol": "parse_line", "line_start": 94, "line_end": 157, "literal_get_calls": [], "dictionary_entries": [ { "line": 146, "key": "I", "value_expression": "1" }, { "line": 146, "key": "V", "value_expression": "5" }, { "line": 146, "key": "X", "value_expression": "10" }, { "line": 155, "key": "year", "value_expression": "year.group(1)" }, { "line": 155, "key": "activity", "value_expression": "activity" }, { "line": 155, "key": "description", "value_expression": "text" }, { "line": 156, "key": "participants", "value_expression": "participants" }, { "line": 156, "key": "technical_facts", "value_expression": "technical" }, { "line": 157, "key": "unresolved", "value_expression": "unresolved" }, { "line": 157, "key": "parser_version", "value_expression": "VERSION" }, { "line": 100, "key": "year", "value_expression": "repair['year']" }, { "line": 100, "key": "activity", "value_expression": "'repair'" }, { "line": 100, "key": "description", "value_expression": "text" }, { "line": 101, "key": "participants", "value_expression": "[{'name': repair['name'], 'activities': ['repairer'], 'role': 'repairer', 'actor_type': 'person', 'place_context': repair['place'], 'evidence_spans': [span(text, *repair.span('name'))], 'name_derivation': 'literal', 'resolution_state': 'unresolved_source_mention'}]" }, { "line": 105, "key": "technical_facts", "value_expression": "[]" }, { "line": 105, "key": "unresolved", "value_expression": "[]" }, { "line": 105, "key": "parser_version", "value_expression": "VERSION" }, { "line": 106, "key": "recovered_embedded_date", "value_expression": "True" }, { "line": 123, "key": "building", "value_expression": "'builder'" }, { "line": 123, "key": "repair", "value_expression": "'repairer'" }, { "line": 124, "key": "restoration", "value_expression": "'restorer'" }, { "line": 124, "key": "rebuilding", "value_expression": "'rebuilder'" }, { "line": 150, "key": "kind", "value_expression": "'manual_stop_count'" }, { "line": 150, "key": "manual_count", "value_expression": "manuals" }, { "line": 151, "key": "stop_count", "value_expression": "int(match.group(2))" }, { "line": 151, "key": "evidence_span", "value_expression": "span(text, *match.span())" }, { "line": 152, "key": "scope", "value_expression": "'documented_event_not_current_state'" }, { "line": 154, "key": "reason", "value_expression": "'no_supported_named_participant'" }, { "line": 101, "key": "name", "value_expression": "repair['name']" }, { "line": 101, "key": "activities", "value_expression": "['repairer']" }, { "line": 101, "key": "role", "value_expression": "'repairer'" }, { "line": 102, "key": "actor_type", "value_expression": "'person'" }, { "line": 102, "key": "place_context", "value_expression": "repair['place']" }, { "line": 103, "key": "evidence_spans", "value_expression": "[span(text, *repair.span('name'))]" }, { "line": 103, "key": "name_derivation", "value_expression": "'literal'" }, { "line": 104, "key": "resolution_state", "value_expression": "'unresolved_source_mention'" } ] }, { "symbol": "parse_history", "line_start": 160, "line_end": 170, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/indo_event_participants.py", "sha256": "743a8df6674bf64c784af4583952731f9ff7c241f869335997260a17cedfce3e", "functions": [ { "symbol": "mentions", "line_start": 8, "line_end": 31, "literal_get_calls": [], "dictionary_entries": [ { "line": 27, "key": "wording", "value_expression": "text" }, { "line": 27, "key": "start", "value_expression": "start" }, { "line": 27, "key": "end", "value_expression": "start + len(text)" }, { "line": 28, "key": "fieldLabel", "value_expression": "match.group(1)" }, { "line": 29, "key": "actorType", "value_expression": "'person' if match.group(1) == 'Facteur(s)' else 'organization'" }, { "line": 30, "key": "status", "value_expression": "'bounded_source_mention' if bounded else 'unresolved_attribution_field'" } ] } ] }, { "path": "aggregator/swiss_specification_accounts.py", "sha256": "2dac0c6d3e0d1ff0ce63a7666fa82af87aa91c5d9f56d32e240e46b30c5541af", "functions": [ { "symbol": "_sha", "line_start": 27, "line_end": 28, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_norm", "line_start": 31, "line_end": 32, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_coord", "line_start": 80, "line_end": 82, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "_rows", "line_start": 85, "line_end": 106, "literal_get_calls": [ { "line": 89, "key": "style", "expression": "line['attrs'].get('style', '')" }, { "line": 101, "key": "style", "expression": "line['attrs'].get('style', '')" } ], "dictionary_entries": [ { "line": 97, "key": "blockIndex", "value_expression": "block['index']" }, { "line": 97, "key": "lineIndex", "value_expression": "line['index']" }, { "line": 98, "key": "sourceSelector", "value_expression": "f\"div.xr_txt[{block['index']}]/span.xr_tl[{line['index']}]\"" }, { "line": 99, "key": "selectorConvention", "value_expression": "'zero_based_matching_descendants'" }, { "line": 100, "key": "htmlStart", "value_expression": "line['start']" }, { "line": 100, "key": "htmlEnd", "value_expression": "line['end']" }, { "line": 100, "key": "offsetUnit", "value_expression": "'unicode_codepoints'" }, { "line": 101, "key": "style", "value_expression": "line['attrs'].get('style', '')" }, { "line": 101, "key": "wording", "value_expression": "text" }, { "line": 105, "key": "wording", "value_expression": "text" }, { "line": 105, "key": "top", "value_expression": "y" }, { "line": 105, "key": "sourceLocators", "value_expression": "locators" } ] }, { "symbol": "_date", "line_start": 109, "line_end": 122, "literal_get_calls": [], "dictionary_entries": [ { "line": 113, "key": "wording", "value_expression": "wording" }, { "line": 113, "key": "expression", "value_expression": "expression" }, { "line": 114, "key": "sourceLocators", "value_expression": "row['sourceLocators']" }, { "line": 114, "key": "presentConditionEstablished", "value_expression": "False" }, { "line": 115, "key": "sourcePath", "value_expression": "'source_snapshot.raw_content'" }, { "line": 116, "key": "basis", "value_expression": "'stand_line_inside_same_bounded_stoplist_account'" }, { "line": 121, "key": "year", "value_expression": "int(match['year'])" }, { "line": 121, "key": "qualification", "value_expression": "qualification" }, { "line": 122, "key": "status", "value_expression": "'explicit_source_configuration_date'" }, { "line": 118, "key": "year", "value_expression": "None" }, { "line": 118, "key": "qualification", "value_expression": "expression" }, { "line": 119, "key": "status", "value_expression": "'unresolved_date_expression'" } ] }, { "symbol": "extract_accounts", "line_start": 125, "line_end": 226, "literal_get_calls": [ { "line": 209, "key": "style", "expression": "root_attributes.get('style', '')" } ], "dictionary_entries": [ { "line": 133, "key": "contract", "value_expression": "CONTRACT" }, { "line": 133, "key": "accounts", "value_expression": "[]" }, { "line": 133, "key": "outcomes", "value_expression": "[]" }, { "line": 137, "key": "blockIndex", "value_expression": "block['index']" }, { "line": 199, "key": "sourcePath", "value_expression": "'source_snapshot.raw_content'" }, { "line": 199, "key": "blockIndex", "value_expression": "block['index']" }, { "line": 199, "key": "sourceSelector", "value_expression": "f\"div.xr_txt[{block['index']}]\"" }, { "line": 200, "key": "selectorConvention", "value_expression": "'zero_based_matching_descendants'" }, { "line": 201, "key": "htmlStart", "value_expression": "block['start']" }, { "line": 201, "key": "htmlEnd", "value_expression": "end" }, { "line": 201, "key": "offsetUnit", "value_expression": "'unicode_codepoints'" }, { "line": 202, "key": "htmlSha256", "value_expression": "_sha(html[block['start']:end])" }, { "line": 202, "key": "blockSha256", "value_expression": "_sha(text)" }, { "line": 203, "key": "rowLocators", "value_expression": "[row['sourceLocators'] for row in rows]" }, { "line": 216, "key": "accountKey", "value_expression": "key" }, { "line": 216, "key": "heading", "value_expression": "rows[0]['wording']" }, { "line": 217, "key": "dateClaim", "value_expression": "_date(date_rows[0]) if date_rows else None" }, { "line": 218, "key": "dateStatus", "value_expression": "'account_stand_claim' if date_rows else 'configuration_date_not_established'" }, { "line": 219, "key": "components", "value_expression": "components" }, { "line": 219, "key": "stopCount", "value_expression": "stop_count" }, { "line": 220, "key": "blockText", "value_expression": "text" }, { "line": 220, "key": "blockSha256", "value_expression": "_sha(text)" }, { "line": 220, "key": "sourceBlocks", "value_expression": "[source_block]" }, { "line": 221, "key": "contextRows", "value_expression": "context" }, { "line": 221, "key": "unresolvedRows", "value_expression": "unresolved" }, { "line": 222, "key": "qualification", "value_expression": "'recovered_source_account; completeness_not_asserted'" }, { "line": 225, "key": "status", "value_expression": "'recovered_account'" }, { "line": 225, "key": "accountKey", "value_expression": "key" }, { "line": 139, "key": "status", "value_expression": "'unpositioned_lines'" }, { "line": 143, "key": "status", "value_expression": "'no_bounded_manual_or_pedal_heading'" }, { "line": 147, "key": "status", "value_expression": "'division_fragment_without_account_heading'" }, { "line": 150, "key": "status", "value_expression": "'multiple_stand_dates_in_one_block'" }, { "line": 155, "key": "status", "value_expression": "'repeated_divisions_require_account_boundary'" }, { "line": 192, "key": "status", "value_expression": "'insufficient_recoverable_stops'" }, { "line": 192, "key": "stopCount", "value_expression": "stop_count" }, { "line": 197, "key": "status", "value_expression": "'unclosed_source_block'" }, { "line": 214, "key": "status", "value_expression": "'identical_layout_copy'" }, { "line": 214, "key": "accountKey", "value_expression": "key" }, { "line": 170, "key": "kind", "value_expression": "'division'" }, { "line": 170, "key": "name", "value_expression": "text" }, { "line": 170, "key": "division", "value_expression": "text" }, { "line": 178, "key": "reason", "value_expression": "'account_accessory_or_context_row'" }, { "line": 184, "key": "kind", "value_expression": "'stop'" }, { "line": 184, "key": "name", "value_expression": "name" }, { "line": 184, "key": "division", "value_expression": "division" }, { "line": 185, "key": "pitch", "value_expression": "match['pitch']" }, { "line": 185, "key": "annotation", "value_expression": "text[match.end('pitch'):].strip()" }, { "line": 167, "key": "reason", "value_expression": "'division_after_account_end'" }, { "line": 187, "key": "kind", "value_expression": "'accessory'" }, { "line": 187, "key": "name", "value_expression": "text" }, { "line": 187, "key": "division", "value_expression": "division" }, { "line": 189, "key": "reason", "value_expression": "'unrecognized_stoplist_row'" } ] }, { "symbol": "__init__", "line_start": 36, "line_end": 44, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "absolute_offset", "line_start": 46, "line_end": 48, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "handle_starttag", "line_start": 50, "line_end": 64, "literal_get_calls": [ { "line": 52, "key": "is_block", "expression": "n.get('is_block')" }, { "line": 54, "key": "class", "expression": "attrs.get('class', '')" }, { "line": 58, "key": "class", "expression": "attrs.get('class', '')" }, { "line": 60, "key": "is_line", "expression": "n.get('is_line')" } ], "dictionary_entries": [ { "line": 53, "key": "tag", "value_expression": "tag" }, { "line": 53, "key": "attrs", "value_expression": "attrs" }, { "line": 53, "key": "start", "value_expression": "self.absolute_offset()" }, { "line": 53, "key": "end", "value_expression": "None" }, { "line": 54, "key": "text", "value_expression": "[]" }, { "line": 54, "key": "is_block", "value_expression": "tag == 'div' and 'xr_txt' in attrs.get('class', '').split()" } ] }, { "symbol": "handle_endtag", "line_start": 66, "line_end": 73, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "handle_data", "line_start": 75, "line_end": 77, "literal_get_calls": [], "dictionary_entries": [] } ] }, { "path": "aggregator/swiss_configuration_date.py", "sha256": "39389db65a663526d27799ce5b5467d57f1a93e1f19ed82c4113b004aabce680", "functions": [ { "symbol": "has_class", "line_start": 33, "line_end": 34, "literal_get_calls": [ { "line": 34, "key": "class", "expression": "node['attrs'].get('class')" } ], "dictionary_entries": [] }, { "symbol": "claims", "line_start": 37, "line_end": 61, "literal_get_calls": [ { "line": 38, "key": "name", "expression": "x.get('name')" }, { "line": 38, "key": "stops", "expression": "specification.get('stops')" }, { "line": 38, "key": "name", "expression": "x.get('name')" }, { "line": 59, "key": "style", "expression": "line['attrs'].get('style', '')" } ], "dictionary_entries": [ { "line": 57, "key": "contract", "value_expression": "CONTRACT" }, { "line": 57, "key": "year", "value_expression": "int(match['year'])" }, { "line": 57, "key": "qualification", "value_expression": "match['qualifier'] or ''" }, { "line": 58, "key": "wording", "value_expression": "wording" }, { "line": 58, "key": "expression", "value_expression": "wording" }, { "line": 58, "key": "sourcePath", "value_expression": "'source_snapshot.raw_content'" }, { "line": 59, "key": "sourceSelector", "value_expression": "'div.xr_txt span.xr_tl'" }, { "line": 59, "key": "dateLineStyle", "value_expression": "line['attrs'].get('style', '')" }, { "line": 60, "key": "basis", "value_expression": "'single_stand_line_in_block_containing_all_transferred_stops'" }, { "line": 61, "key": "presentConditionEstablished", "value_expression": "False" } ] }, { "symbol": "__init__", "line_start": 10, "line_end": 13, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "handle_starttag", "line_start": 15, "line_end": 19, "literal_get_calls": [], "dictionary_entries": [ { "line": 16, "key": "tag", "value_expression": "tag" }, { "line": 16, "key": "attrs", "value_expression": "dict(attrs)" }, { "line": 16, "key": "parent", "value_expression": "self.stack[-1] if self.stack else None" }, { "line": 16, "key": "text", "value_expression": "[]" } ] }, { "symbol": "handle_endtag", "line_start": 21, "line_end": 25, "literal_get_calls": [], "dictionary_entries": [] }, { "symbol": "handle_data", "line_start": 27, "line_end": 30, "literal_get_calls": [], "dictionary_entries": [] } ] } ], "reference_ids": [ "selection_scope", "atlas_datasheets", "atlas_prov", "code_35", "code_32", "aeolianskinner_main", "record_aeolianskinner", "arpschnitger", "record_arpschnitger", "augustine_virtual_organs_samples", "code_09", "code_40", "bdo_tuning_systems", "bdo_authors", "code_02", "code_36", "binauralpipes_instruments", "contrebombarde_organs", "digital_sheet_music", "repo_score_collections", "code_03", "code_04", "estey", "record_estey", "evensong_hauptwerk_samplesets", "evensong_rights", "gdo_intro", "gdo_literature", "code_06", "code_07", "gdo_orgelbauer_links", "code_08", "code_12", "grandorgue_github_wiki", "indo", "record_indo", "code_10", "iohio", "iohio_home", "record_iohio", "code_13", "jeux_d_orgues_downloads", "kirchenorgelforum", "code_14", "code_05", "lars_palo_vpo", "lavender_audio_hereford", "indexed_milan", "milan_digital_audio_instruments", "mps_orgelseite_filebase", "musiqueorguequebec", "record_musiqueorguequebec", "browser_musixplora", "code_15", "code_16", "code_18", "code_17", "nomine", "record_nomine", "code_19", "browser_npor", "rendered_record_npor", "code_20", "indexed_nycago", "nycago_catalogue", "record_nycago", "code_21", "web_ohs", "ohs", "rendered_record_ohs", "code_22", "code_24", "code_23", "ohta", "ohta_rights", "record_ohta", "organindex", "record_organindex", "code_25", "orgbase", "orgbase_about", "orgbase_dbis", "record_orgbase", "code_26", "orgelarchiv_schmidt", "orgelarchiv_about", "record_orgelarchiv_schmidt", "code_31", "orgeldb_bayern", "record_orgeldb_bayern", "orgeldoc", "hslu_status", "record_orgeldoc", "orgelsachsen", "record_orgelsachsen", "code_27", "orgelseite", "orgelseite_imprint", "record_orgelseite_list", "code_28", "orgelsite", "record_orgelsite", "orgelverzeichnis_ch", "swiss_rights", "record_orgelverzeichnis_ch", "orguesch", "record_orguesch", "orguesfrance", "record_orguesfrance", "oswa", "record_oswa", "pcorgan_hauptwerk_samplesets", "pcorgan_about", "piotr_grabowski_go_versions", "piotr_grabowski_instruments", "pipedreams", "record_pipedreams", "pipeorganmap", "record_pipeorganmap", "ridim_iconography", "ridim_imprint", "code_29", "code_30", "sakralorgelforum", "browser_sonora", "sonora_project", "record_sonora", "sonus_paradisi_free_stuff", "sourceforge_ourorgan_samplesets", "varhany", "varhany_about", "record_varhany", "code_37", "vi_control", "code_38", "code_39", "virtual_pipe_organs_free_producers", "vpo_about", "virtual_pipe_organs_odfs", "score_pdmx", "score_orgelbuechlein", "score_imslp_wima", "score_mutopia", "mutopia_rights", "score_musescore_direct", "audit_orkasa", "repo_audit", "audit_organstops", "audit_welt_orgel_archiv", "code_11", "code_34", "code_33" ] }