piedomains

• piedomains package
  • Subpackages
    • piedomains.llm package
      • Submodules
      • piedomains.llm.config module
      • piedomains.llm.prompts module
      • piedomains.llm.response_parser module
      • Module contents
  • Submodules
  • piedomains.api module
    • DomainClassifier
      • DomainClassifier.classify()
      • DomainClassifier.classify_by_text()
      • DomainClassifier.classify_by_images()
      • DomainClassifier.configure_llm()
      • DomainClassifier.classify_by_llm()
      • DomainClassifier.classify_by_llm_multimodal()
      • DomainClassifier.get_llm_usage_stats()
      • DomainClassifier.collect_content()
      • DomainClassifier.classify_from_collection()
    • classify_domains()
  • piedomains.cli module
    • build_parser()
    • training_scripts_dir()
    • main()
  • piedomains.config module
    • Config
      • Config.DEFAULT_CONFIG
      • Config.get()
      • Config.set()
      • Config.update()
      • Config.to_dict()
      • Config.http_timeout
      • Config.webdriver_timeout
      • Config.page_load_timeout
      • Config.max_retries
      • Config.retry_delay
      • Config.screenshot_wait_time
      • Config.webdriver_window_size
      • Config.batch_size
      • Config.parallel_workers
      • Config.user_agent
      • Config.image_size
      • Config.enable_content_validation
      • Config.content_safety_mode
      • Config.max_content_length
      • Config.sandbox_mode_required
      • Config.allowed_content_types
      • Config.blocked_extensions
      • Config.suspicious_url_patterns
      • Config.content_length_limits
    • get_config()
    • set_config()
    • configure()
  • piedomains.constants module
    • classes
    • most_common_words
    • get_valid_categories()
    • is_valid_category()
    • get_category_count()
  • piedomains.content_processor module
    • ContentProcessor
      • ContentProcessor.extract_all_content()
      • ContentProcessor.extract_html_content()
      • ContentProcessor.extract_text_content()
      • ContentProcessor.extract_image_content()
      • ContentProcessor.prepare_image_tensors()
  • piedomains.content_validation module
    • ContentValidationResult
      • ContentValidationResult.is_safe
      • ContentValidationResult.content_type
      • ContentValidationResult.content_length
      • ContentValidationResult.error_message
      • ContentValidationResult.warnings
      • ContentValidationResult.sandbox_recommended
    • ContentValidator
      • ContentValidator.validate_url()
      • ContentValidator.validate_url_offline()
      • ContentValidator.get_sandbox_command()
  • piedomains.context_managers module
    • webdriver_context()
    • playwright_context()
    • temporary_directory()
    • file_cleanup()
    • error_recovery()
    • batch_progress_tracking()
    • ResourceManager
      • ResourceManager.add_driver()
      • ResourceManager.add_fetcher()
      • ResourceManager.add_temp_directory()
      • ResourceManager.add_temp_file()
      • ResourceManager.cleanup_all()
  • piedomains.data_collector module
    • DataCollector
      • DataCollector.collect()
      • DataCollector.collect_batch()
      • DataCollector.load_collection()
      • DataCollector.list_collections()
  • piedomains.fetchers module
    • FetchResult
      • FetchResult.url
      • FetchResult.success
      • FetchResult.html
      • FetchResult.text
      • FetchResult.screenshot_path
      • FetchResult.title
      • FetchResult.meta_description
      • FetchResult.error
      • FetchResult.error_code
      • FetchResult.snapshot_timestamp
      • FetchResult.source
    • BaseFetcher
      • BaseFetcher.fetch_single()
      • BaseFetcher.fetch_batch()
      • BaseFetcher.cleanup()
      • BaseFetcher.fetch_both()
    • PlaywrightFetcher
      • PlaywrightFetcher.fetch_single()
      • PlaywrightFetcher.fetch_batch()
      • PlaywrightFetcher.fetch_html()
      • PlaywrightFetcher.fetch_content()
      • PlaywrightFetcher.fetch_screenshot()
    • ArchiveFetcher
      • ArchiveFetcher.iframe_url()
      • ArchiveFetcher.find_closest_record()
      • ArchiveFetcher.fetch_single()
      • ArchiveFetcher.fetch_batch()
      • ArchiveFetcher.fetch_html()
    • get_fetcher()
  • piedomains.http_client module
    • PooledHTTPClient
      • PooledHTTPClient.session
      • PooledHTTPClient.get()
      • PooledHTTPClient.close()
    • http_client()
    • get_http_client()
    • close_global_client()
  • piedomains.image module
    • DEFAULT_IMAGE_MODEL
    • resolve_image_model()
    • ImageClassifier
      • ImageClassifier.load_models()
      • ImageClassifier.predict_proba()
      • ImageClassifier.classify()
      • ImageClassifier.classify_from_paths()
      • ImageClassifier.classify_from_data()
  • piedomains.llm_classifier module
    • LLMClassifier
      • LLMClassifier.get_usage_stats()
      • LLMClassifier.reset_usage_stats()
      • LLMClassifier.classify_text()
      • LLMClassifier.classify_multimodal()
      • LLMClassifier.classify_from_paths()
      • LLMClassifier.classify_from_data()
  • piedomains.piedomain module
    • Piedomain
      • Piedomain.weights_loaded
      • Piedomain.img_width
      • Piedomain.img_height
      • Piedomain.parse_url_to_domain()
      • Piedomain.validate_url_or_domain()
      • Piedomain.validate_domain_name()
      • Piedomain.validate_domains()
      • Piedomain.validate_urls_or_domains()
      • Piedomain.text_from_html()
      • Piedomain.data_cleanup()
      • Piedomain.validate_input()
  • piedomains.piedomains_logging module
    • JsonFormatter
      • JsonFormatter.format()
    • bind_context()
    • clear_context()
    • get_logger()
    • configure_logging()
    • set_level()
    • get_effective_level()
    • disable_logging()
    • is_debug_enabled()
  • piedomains.text_processor module
    • TextProcessor
      • TextProcessor.extract_text_from_html()
      • TextProcessor.clean_and_normalize_text()
      • TextProcessor.process_html_to_text()
      • TextProcessor.extract_with_trafilatura()
      • TextProcessor.extract_text()
  • piedomains.text module
    • DEFAULT_TEXT_MODEL
    • resolve_text_model()
    • TextClassifier
      • TextClassifier.load_models()
      • TextClassifier.classify()
      • TextClassifier.classify_from_paths()
      • TextClassifier.classify_from_data()
  • piedomains.utils module
    • REPO_BASE_URL
    • download_file()
    • is_within_directory()
    • safe_extract()
    • SecurityError
    • get_file_hash()
  • Module contents