Index

_

__alias (nyawc.helpers.PackageHelper.PackageHelper attribute) __cache (nyawc.helpers.URLHelper.URLHelper attribute) __callback (nyawc.CrawlerThread.CrawlerThread attribute) __callback_lock (nyawc.CrawlerThread.CrawlerThread attribute) __description (nyawc.helpers.PackageHelper.PackageHelper attribute) __index_hash (nyawc.QueueItem.QueueItem attribute) __init__() (nyawc.Crawler.Crawler method) (nyawc.CrawlerThread.CrawlerThread method) (nyawc.Options.Options method) (nyawc.Options.OptionsCallbacks method) (nyawc.Options.OptionsIdentity method) (nyawc.Options.OptionsMisc method) (nyawc.Options.OptionsPerformance method) (nyawc.Options.OptionsRouting method) (nyawc.Options.OptionsScope method) (nyawc.Queue.Queue method) (nyawc.QueueItem.QueueItem method) (nyawc.Routing.Routing method) (nyawc.http.Handler.Handler method) (nyawc.http.Request.Request method) (nyawc.http.Response.Response method) (nyawc.scrapers.BaseScraper.BaseScraper method) __lock (nyawc.Crawler.Crawler attribute) __name (nyawc.helpers.PackageHelper.PackageHelper attribute) __options (nyawc.Crawler.Crawler attribute) (nyawc.CrawlerThread.CrawlerThread attribute) (nyawc.Queue.Queue attribute) (nyawc.http.Handler.Handler attribute) (nyawc.scrapers.BaseScraper.BaseScraper attribute) __queue_item (nyawc.CrawlerThread.CrawlerThread attribute) (nyawc.http.Handler.Handler attribute) (nyawc.scrapers.BaseScraper.BaseScraper attribute) __response_soup (nyawc.QueueItem.QueueItem attribute) __routing_count (nyawc.Routing.Routing attribute) __routing_options (nyawc.Routing.Routing attribute) __should_spawn_new_requests (nyawc.Crawler.Crawler attribute) __should_stop (nyawc.Crawler.Crawler attribute)

__stopped (nyawc.Crawler.Crawler attribute) __stopping (nyawc.Crawler.Crawler attribute) __threads (nyawc.Crawler.Crawler attribute) __version (nyawc.helpers.PackageHelper.PackageHelper attribute) _Crawler__add_scraped_requests_to_queue() (nyawc.Crawler.Crawler method) _Crawler__crawler_finish() (nyawc.Crawler.Crawler method) _Crawler__crawler_start() (nyawc.Crawler.Crawler method) _Crawler__crawler_stop() (nyawc.Crawler.Crawler method) _Crawler__request_finish() (nyawc.Crawler.Crawler method) _Crawler__request_start() (nyawc.Crawler.Crawler method) _Crawler__signal_handler() (nyawc.Crawler.Crawler method) _Crawler__spawn_new_request() (nyawc.Crawler.Crawler method) _Crawler__spawn_new_requests() (nyawc.Crawler.Crawler method) _Crawler__wait_for_current_threads() (nyawc.Crawler.Crawler method) _Handler__content_type_matches() (nyawc.http.Handler.Handler method) _Handler__get_all_scrapers() (nyawc.http.Handler.Handler method) _Handler__get_all_scrapers_modules() (nyawc.http.Handler.Handler method) _Handler__make_request() (nyawc.http.Handler.Handler method) _HTMLSoupFormScraper__autofill_form_data() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupFormScraper__get_default_form_data_input() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupFormScraper__get_default_value_from_element() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupFormScraper__get_form_data() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupFormScraper__get_request() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupFormScraper__get_valid_form_data_elements() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupFormScraper__trim_grave_accent() (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) _HTMLSoupLinkScraper__trim_grave_accent() (nyawc.scrapers.HTMLSoupLinkScraper.HTMLSoupLinkScraper method) _OptionsCallbacks__null_route_crawler_after_finish() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_crawler_before_start() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_form_after_autofill() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_form_before_autofill() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_request_after_finish() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_request_before_start() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_request_in_thread_after_finish() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_request_in_thread_before_start() (nyawc.Options.OptionsCallbacks method) _OptionsCallbacks__null_route_request_on_error() (nyawc.Options.OptionsCallbacks method) _Queue__get_var() (nyawc.Queue.Queue method) _Queue__set_var() (nyawc.Queue.Queue method)

A

add() (nyawc.Queue.Queue method) add_request() (nyawc.Queue.Queue method)

append_with_data() (nyawc.helpers.URLHelper.URLHelper static method) auth (nyawc.http.Request.Request attribute) (nyawc.Options.OptionsIdentity attribute)

B

BaseScraper (class in nyawc.scrapers.BaseScraper)

C

cache (nyawc.helpers.RandomInputHelper.RandomInputHelper attribute), [1] callbacks (nyawc.Options.Options attribute) complies_with_scope() (nyawc.helpers.HTTPRequestHelper.HTTPRequestHelper static method) content_types (nyawc.scrapers.CSSRegexLinkScraper.CSSRegexLinkScraper attribute) (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper attribute) (nyawc.scrapers.HTMLSoupLinkScraper.HTMLSoupLinkScraper attribute) (nyawc.scrapers.JSONRegexLinkScraper.JSONRegexLinkScraper attribute) (nyawc.scrapers.XMLRegexLinkScraper.XMLRegexLinkScraper attribute)

cookies (nyawc.http.Request.Request attribute) (nyawc.Options.OptionsIdentity attribute) count_total (nyawc.Queue.Queue attribute) Crawler (class in nyawc.Crawler) crawler_after_finish (nyawc.Options.OptionsCallbacks attribute) crawler_before_start (nyawc.Options.OptionsCallbacks attribute) CrawlerActions (class in nyawc.CrawlerActions) CrawlerThread (class in nyawc.CrawlerThread) CSSRegexLinkScraper (class in nyawc.scrapers.CSSRegexLinkScraper)

D

data (nyawc.http.Request.Request attribute) debug (nyawc.Options.OptionsMisc attribute) DebugHelper (class in nyawc.helpers.DebugHelper) decompose() (nyawc.QueueItem.QueueItem method) decomposed (nyawc.QueueItem.QueueItem attribute) depth (nyawc.http.Request.Request attribute) derived_get_requests() (nyawc.scrapers.CSSRegexLinkScraper.CSSRegexLinkScraper method) (nyawc.scrapers.HTMLSoupFormScraper.HTMLSoupFormScraper method) (nyawc.scrapers.HTMLSoupLinkScraper.HTMLSoupLinkScraper method) (nyawc.scrapers.JSONRegexLinkScraper.JSONRegexLinkScraper method) (nyawc.scrapers.XMLRegexLinkScraper.XMLRegexLinkScraper method)

DO_AUTOFILL_FORM (nyawc.CrawlerActions.CrawlerActions attribute), [1] DO_CONTINUE_CRAWLING (nyawc.CrawlerActions.CrawlerActions attribute), [1] DO_NOT_AUTOFILL_FORM (nyawc.CrawlerActions.CrawlerActions attribute), [1] DO_SKIP_TO_NEXT (nyawc.CrawlerActions.CrawlerActions attribute), [1] DO_STOP_CRAWLING (nyawc.CrawlerActions.CrawlerActions attribute), [1]

F

form_after_autofill (nyawc.Options.OptionsCallbacks attribute)

form_before_autofill (nyawc.Options.OptionsCallbacks attribute)

G

get_alias() (nyawc.helpers.PackageHelper.PackageHelper static method) get_all() (nyawc.Queue.Queue method) get_cookie_header() (nyawc.helpers.HTTPRequestHelper.HTTPRequestHelper static method) get_description() (nyawc.helpers.PackageHelper.PackageHelper static method) get_first() (nyawc.Queue.Queue method) get_for_type() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_hash() (nyawc.QueueItem.QueueItem method) get_hostname() (nyawc.helpers.URLHelper.URLHelper static method) get_name() (nyawc.helpers.PackageHelper.PackageHelper static method) get_new_requests() (nyawc.http.Handler.Handler method) get_ordered_params() (nyawc.helpers.URLHelper.URLHelper static method) get_path() (nyawc.helpers.URLHelper.URLHelper static method) get_progress() (nyawc.Queue.Queue method)

get_protocol() (nyawc.helpers.URLHelper.URLHelper static method) get_random_color() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_email() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_number() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_password() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_telephonenumber() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_text() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_url() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_random_value() (nyawc.helpers.RandomInputHelper.RandomInputHelper static method) get_requests() (nyawc.scrapers.BaseScraper.BaseScraper method) get_soup_response() (nyawc.QueueItem.QueueItem method) get_subdomain() (nyawc.helpers.URLHelper.URLHelper static method) get_tld() (nyawc.helpers.URLHelper.URLHelper static method) get_version() (nyawc.helpers.PackageHelper.PackageHelper static method)

H

Handler (class in nyawc.http.Handler) has_request() (nyawc.Queue.Queue method) headers (nyawc.http.Request.Request attribute) (nyawc.Options.OptionsIdentity attribute)

hostname_must_match (nyawc.Options.OptionsScope attribute) HTMLSoupFormScraper (class in nyawc.scrapers.HTMLSoupFormScraper) HTMLSoupLinkScraper (class in nyawc.scrapers.HTMLSoupLinkScraper) HTTPRequestHelper (class in nyawc.helpers.HTTPRequestHelper)

I

identity (nyawc.Options.Options attribute) increase_route_count() (nyawc.Routing.Routing method)

is_mailto() (nyawc.helpers.URLHelper.URLHelper static method) is_parsable() (nyawc.helpers.URLHelper.URLHelper static method) is_treshold_reached() (nyawc.Routing.Routing method)

J

JSONRegexLinkScraper (class in nyawc.scrapers.JSONRegexLinkScraper)

M

make_absolute() (nyawc.helpers.URLHelper.URLHelper static method) max_depth (nyawc.Options.OptionsScope attribute) max_threads (nyawc.Options.OptionsPerformance attribute) method (nyawc.http.Request.Request attribute) METHOD_DELETE (nyawc.http.Request.Request attribute), [1] METHOD_GET (nyawc.http.Request.Request attribute), [1] METHOD_HEAD (nyawc.http.Request.Request attribute), [1]

METHOD_OPTIONS (nyawc.http.Request.Request attribute), [1] METHOD_POST (nyawc.http.Request.Request attribute), [1] METHOD_PUT (nyawc.http.Request.Request attribute), [1] minimum_threshold (nyawc.Options.OptionsRouting attribute) misc (nyawc.Options.Options attribute) move() (nyawc.Queue.Queue method) move_bulk() (nyawc.Queue.Queue method)

N

nyawc (module) nyawc.Crawler (module) nyawc.CrawlerActions (module) nyawc.CrawlerThread (module) nyawc.helpers (module) nyawc.helpers.DebugHelper (module) nyawc.helpers.HTTPRequestHelper (module) nyawc.helpers.PackageHelper (module) nyawc.helpers.RandomInputHelper (module) nyawc.helpers.URLHelper (module) nyawc.http (module) nyawc.http.Handler (module)

nyawc.http.Request (module) nyawc.http.Response (module) nyawc.Options (module) nyawc.Queue (module) nyawc.QueueItem (module) nyawc.Routing (module) nyawc.scrapers (module) nyawc.scrapers.BaseScraper (module) nyawc.scrapers.CSSRegexLinkScraper (module) nyawc.scrapers.HTMLSoupFormScraper (module) nyawc.scrapers.HTMLSoupLinkScraper (module) nyawc.scrapers.JSONRegexLinkScraper (module) nyawc.scrapers.XMLRegexLinkScraper (module)

O

Options (class in nyawc.Options) OptionsCallbacks (class in nyawc.Options) OptionsIdentity (class in nyawc.Options) OptionsMisc (class in nyawc.Options)

OptionsPerformance (class in nyawc.Options) OptionsRouting (class in nyawc.Options) OptionsScope (class in nyawc.Options) output() (nyawc.helpers.DebugHelper.DebugHelper static method)

P

PackageHelper (class in nyawc.helpers.PackageHelper) parent_raised_error (nyawc.http.Request.Request attribute) patch_with_options() (nyawc.helpers.HTTPRequestHelper.HTTPRequestHelper static method)

performance (nyawc.Options.Options attribute) protocol_must_match (nyawc.Options.OptionsScope attribute) proxies (nyawc.http.Request.Request attribute) (nyawc.Options.OptionsIdentity attribute)

Q

query_dict_to_string() (nyawc.helpers.URLHelper.URLHelper static method) query_string_to_dict() (nyawc.helpers.URLHelper.URLHelper static method)

Queue (class in nyawc.Queue) queue (nyawc.Crawler.Crawler attribute) QueueItem (class in nyawc.QueueItem)

R

RandomInputHelper (class in nyawc.helpers.RandomInputHelper) remove_hash() (nyawc.helpers.URLHelper.URLHelper static method) Request (class in nyawc.http.Request) request (nyawc.QueueItem.QueueItem attribute) request_after_finish (nyawc.Options.OptionsCallbacks attribute) request_before_start (nyawc.Options.OptionsCallbacks attribute) request_in_thread_after_finish (nyawc.Options.OptionsCallbacks attribute) request_in_thread_before_start (nyawc.Options.OptionsCallbacks attribute) request_on_error (nyawc.Options.OptionsCallbacks attribute)

request_timeout (nyawc.Options.OptionsPerformance attribute) Response (class in nyawc.http.Response) response (nyawc.QueueItem.QueueItem attribute) routes (nyawc.Options.OptionsRouting attribute) Routing (class in nyawc.Routing) routing (nyawc.Crawler.Crawler attribute) (nyawc.Options.Options attribute) rst_to_pypi() (nyawc.helpers.PackageHelper.PackageHelper static method) run() (nyawc.CrawlerThread.CrawlerThread method)

S

scope (nyawc.Options.Options attribute) setup() (nyawc.helpers.DebugHelper.DebugHelper static method) start_with() (nyawc.Crawler.Crawler method) status (nyawc.QueueItem.QueueItem attribute) STATUS_CANCELLED (nyawc.QueueItem.QueueItem attribute), [1]

STATUS_ERRORED (nyawc.QueueItem.QueueItem attribute), [1] STATUS_FINISHED (nyawc.QueueItem.QueueItem attribute), [1] STATUS_IN_PROGRESS (nyawc.QueueItem.QueueItem attribute), [1] STATUS_QUEUED (nyawc.QueueItem.QueueItem attribute), [1] STATUSES (nyawc.QueueItem.QueueItem attribute), [1] subdomain_must_match (nyawc.Options.OptionsScope attribute)

T

timeout (nyawc.http.Request.Request attribute)

tld_must_match (nyawc.Options.OptionsScope attribute) trusted_certificates (nyawc.Options.OptionsMisc attribute)

U

url (nyawc.http.Request.Request attribute) (nyawc.http.Response.Response attribute)

URLHelper (class in nyawc.helpers.URLHelper)

V

verify (nyawc.http.Request.Request attribute)

verify_ssl_certificates (nyawc.Options.OptionsMisc attribute)

X

XMLRegexLinkScraper (class in nyawc.scrapers.XMLRegexLinkScraper)