Index

A

add_driver() (piedomains.context_managers.ResourceManager method) add_fetcher() (piedomains.context_managers.ResourceManager method) add_temp_directory() (piedomains.context_managers.ResourceManager method)

add_temp_file() (piedomains.context_managers.ResourceManager method) allowed_content_types (piedomains.config.Config property) api_key (piedomains.llm.config.LLMConfig attribute) ArchiveFetcher (class in piedomains.fetchers)

B

base_url (piedomains.llm.config.LLMConfig attribute) BaseFetcher (class in piedomains.fetchers) batch_progress_tracking() (in module piedomains.context_managers)

batch_size (piedomains.config.Config property) bind_context() (in module piedomains.piedomains_logging) blocked_extensions (piedomains.config.Config property) build_parser() (in module piedomains.cli)

C

categories (piedomains.llm.config.LLMConfig attribute) classes (in module piedomains.constants) classify() (piedomains.api.DomainClassifier method) (piedomains.image.ImageClassifier method) (piedomains.text.TextClassifier method) classify_by_images() (piedomains.api.DomainClassifier method) classify_by_llm() (piedomains.api.DomainClassifier method) classify_by_llm_multimodal() (piedomains.api.DomainClassifier method) classify_by_text() (piedomains.api.DomainClassifier method) classify_domains() (in module piedomains.api) classify_from_collection() (piedomains.api.DomainClassifier method) classify_from_data() (piedomains.image.ImageClassifier method) (piedomains.llm_classifier.LLMClassifier method) (piedomains.text.TextClassifier method) classify_from_paths() (piedomains.image.ImageClassifier method) (piedomains.llm_classifier.LLMClassifier method) (piedomains.text.TextClassifier method) classify_multimodal() (piedomains.llm_classifier.LLMClassifier method) classify_text() (piedomains.llm_classifier.LLMClassifier method) clean_and_normalize_text() (piedomains.text_processor.TextProcessor static method)

cleanup() (piedomains.fetchers.BaseFetcher method) cleanup_all() (piedomains.context_managers.ResourceManager method) clear_context() (in module piedomains.piedomains_logging) close() (piedomains.http_client.PooledHTTPClient method) close_global_client() (in module piedomains.http_client) collect() (piedomains.data_collector.DataCollector method) collect_batch() (piedomains.data_collector.DataCollector method) collect_content() (piedomains.api.DomainClassifier method) Config (class in piedomains.config) configure() (in module piedomains.config) configure_llm() (piedomains.api.DomainClassifier method) configure_logging() (in module piedomains.piedomains_logging) content_length (piedomains.content_validation.ContentValidationResult attribute) content_length_limits (piedomains.config.Config property) content_safety_mode (piedomains.config.Config property) content_type (piedomains.content_validation.ContentValidationResult attribute) ContentProcessor (class in piedomains.content_processor) ContentValidationResult (class in piedomains.content_validation) ContentValidator (class in piedomains.content_validation) cost_limit_usd (piedomains.llm.config.LLMConfig attribute)

D

data_cleanup() (piedomains.piedomain.Piedomain class method) DataCollector (class in piedomains.data_collector) DEFAULT_CONFIG (piedomains.config.Config attribute) DEFAULT_IMAGE_MODEL (in module piedomains.image)

DEFAULT_TEXT_MODEL (in module piedomains.text) disable_logging() (in module piedomains.piedomains_logging) DomainClassifier (class in piedomains.api) download_file() (in module piedomains.utils)

E

enable_content_validation (piedomains.config.Config property) error (piedomains.fetchers.FetchResult attribute) error_code (piedomains.fetchers.FetchResult attribute) error_message (piedomains.content_validation.ContentValidationResult attribute) error_recovery() (in module piedomains.context_managers) extract_all_content() (piedomains.content_processor.ContentProcessor method)

extract_html_content() (piedomains.content_processor.ContentProcessor method) extract_image_content() (piedomains.content_processor.ContentProcessor method) extract_text() (piedomains.text_processor.TextProcessor class method) extract_text_content() (piedomains.content_processor.ContentProcessor method) extract_text_from_html() (piedomains.text_processor.TextProcessor static method) extract_with_trafilatura() (piedomains.text_processor.TextProcessor static method)

F

fetch_batch() (piedomains.fetchers.ArchiveFetcher method) (piedomains.fetchers.BaseFetcher method) (piedomains.fetchers.PlaywrightFetcher method) fetch_both() (piedomains.fetchers.BaseFetcher method) fetch_content() (piedomains.fetchers.PlaywrightFetcher method) fetch_html() (piedomains.fetchers.ArchiveFetcher method) (piedomains.fetchers.PlaywrightFetcher method) fetch_screenshot() (piedomains.fetchers.PlaywrightFetcher method)

fetch_single() (piedomains.fetchers.ArchiveFetcher method) (piedomains.fetchers.BaseFetcher method) (piedomains.fetchers.PlaywrightFetcher method) FetchResult (class in piedomains.fetchers) file_cleanup() (in module piedomains.context_managers) find_closest_record() (piedomains.fetchers.ArchiveFetcher method) format() (piedomains.piedomains_logging.JsonFormatter method) from_dict() (piedomains.llm.config.LLMConfig class method)

G

get() (piedomains.config.Config method) (piedomains.http_client.PooledHTTPClient method) get_batch_prompt() (in module piedomains.llm.prompts) get_category_count() (in module piedomains.constants) get_classification_prompt() (in module piedomains.llm.prompts) get_config() (in module piedomains.config) get_custom_prompt() (in module piedomains.llm.prompts) get_effective_level() (in module piedomains.piedomains_logging)

get_fetcher() (in module piedomains.fetchers) get_file_hash() (in module piedomains.utils) get_http_client() (in module piedomains.http_client) get_llm_usage_stats() (piedomains.api.DomainClassifier method) get_logger() (in module piedomains.piedomains_logging) get_multimodal_prompt() (in module piedomains.llm.prompts) get_sandbox_command() (piedomains.content_validation.ContentValidator method) get_usage_stats() (piedomains.llm_classifier.LLMClassifier method) get_valid_categories() (in module piedomains.constants)

H

html (piedomains.fetchers.FetchResult attribute)

http_client() (in module piedomains.http_client) http_timeout (piedomains.config.Config property)

I

iframe_url() (piedomains.fetchers.ArchiveFetcher method) image_size (piedomains.config.Config property) ImageClassifier (class in piedomains.image) img_height (piedomains.piedomain.Piedomain attribute)

img_width (piedomains.piedomain.Piedomain attribute) is_debug_enabled() (in module piedomains.piedomains_logging) is_safe (piedomains.content_validation.ContentValidationResult attribute) is_valid_category() (in module piedomains.constants) is_within_directory() (in module piedomains.utils)

J

JsonFormatter (class in piedomains.piedomains_logging)

L

list_collections() (piedomains.data_collector.DataCollector method) LLMClassifier (class in piedomains.llm_classifier) LLMConfig (class in piedomains.llm.config)

load_collection() (piedomains.data_collector.DataCollector method) load_models() (piedomains.image.ImageClassifier method) (piedomains.text.TextClassifier method)

M

main() (in module piedomains.cli) max_content_length (piedomains.config.Config property) max_retries (piedomains.config.Config property) max_tokens (piedomains.llm.config.LLMConfig attribute) meta_description (piedomains.fetchers.FetchResult attribute) model (piedomains.llm.config.LLMConfig attribute) module piedomains piedomains.api piedomains.cli piedomains.config piedomains.constants piedomains.content_processor piedomains.content_validation piedomains.context_managers piedomains.data_collector piedomains.fetchers piedomains.http_client piedomains.image piedomains.llm piedomains.llm.config piedomains.llm.prompts piedomains.llm.response_parser piedomains.llm_classifier piedomains.piedomain piedomains.piedomains_logging piedomains.text piedomains.text_processor piedomains.utils

most_common_words (in module piedomains.constants)

P

page_load_timeout (piedomains.config.Config property) parallel_workers (piedomains.config.Config property) parse_batch_response() (in module piedomains.llm.response_parser) parse_llm_response() (in module piedomains.llm.response_parser) parse_url_to_domain() (piedomains.piedomain.Piedomain static method) Piedomain (class in piedomains.piedomain) piedomains module piedomains.api module piedomains.cli module piedomains.config module piedomains.constants module piedomains.content_processor module piedomains.content_validation module piedomains.context_managers module piedomains.data_collector module piedomains.fetchers module piedomains.http_client module

piedomains.image module piedomains.llm module piedomains.llm.config module piedomains.llm.prompts module piedomains.llm.response_parser module piedomains.llm_classifier module piedomains.piedomain module piedomains.piedomains_logging module piedomains.text module piedomains.text_processor module piedomains.utils module playwright_context() (in module piedomains.context_managers) PlaywrightFetcher (class in piedomains.fetchers) PooledHTTPClient (class in piedomains.http_client) predict_proba() (piedomains.image.ImageClassifier method) prepare_image_tensors() (piedomains.content_processor.ContentProcessor method) process_html_to_text() (piedomains.text_processor.TextProcessor class method) provider (piedomains.llm.config.LLMConfig attribute)

R

REPO_BASE_URL (in module piedomains.utils) reset_usage_stats() (piedomains.llm_classifier.LLMClassifier method) resolve_image_model() (in module piedomains.image)

resolve_text_model() (in module piedomains.text) ResourceManager (class in piedomains.context_managers) retry_delay (piedomains.config.Config property)

S

safe_extract() (in module piedomains.utils) sandbox_mode_required (piedomains.config.Config property) sandbox_recommended (piedomains.content_validation.ContentValidationResult attribute) screenshot_path (piedomains.fetchers.FetchResult attribute) screenshot_wait_time (piedomains.config.Config property) SecurityError session (piedomains.http_client.PooledHTTPClient property)

set() (piedomains.config.Config method) set_config() (in module piedomains.config) set_level() (in module piedomains.piedomains_logging) snapshot_timestamp (piedomains.fetchers.FetchResult attribute) source (piedomains.fetchers.FetchResult attribute) success (piedomains.fetchers.FetchResult attribute) suspicious_url_patterns (piedomains.config.Config property)

T

temperature (piedomains.llm.config.LLMConfig attribute) temporary_directory() (in module piedomains.context_managers) text (piedomains.fetchers.FetchResult attribute) text_from_html() (piedomains.piedomain.Piedomain class method) TextClassifier (class in piedomains.text)

TextProcessor (class in piedomains.text_processor) title (piedomains.fetchers.FetchResult attribute) to_dict() (piedomains.config.Config method) to_litellm_params() (piedomains.llm.config.LLMConfig method) training_scripts_dir() (in module piedomains.cli)

U

update() (piedomains.config.Config method) url (piedomains.fetchers.FetchResult attribute)

usage_tracking (piedomains.llm.config.LLMConfig attribute) user_agent (piedomains.config.Config property)

V

validate_domain_name() (piedomains.piedomain.Piedomain static method) validate_domains() (piedomains.piedomain.Piedomain class method) validate_input() (piedomains.piedomain.Piedomain class method)

validate_url() (piedomains.content_validation.ContentValidator method) validate_url_offline() (piedomains.content_validation.ContentValidator method) validate_url_or_domain() (piedomains.piedomain.Piedomain static method) validate_urls_or_domains() (piedomains.piedomain.Piedomain class method)

W

warnings (piedomains.content_validation.ContentValidationResult attribute) webdriver_context() (in module piedomains.context_managers)

webdriver_timeout (piedomains.config.Config property) webdriver_window_size (piedomains.config.Config property) weights_loaded (piedomains.piedomain.Piedomain attribute)