UNPKG

7.2 kBTypeScriptView Raw
1export interface ProxyConfigurationFunction {
2 (sessionId: string | number): string | Promise<string>;
3}
4export interface ProxyConfigurationOptions {
5 /**
6 * An array of custom proxy URLs to be rotated.
7 * Custom proxies are not compatible with Apify Proxy and an attempt to use both
8 * configuration options will cause an error to be thrown on initialize.
9 */
10 proxyUrls?: string[];
11 /**
12 * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
13 * and should always return stringified proxy URL. Can be asynchronous.
14 * This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.
15 */
16 newUrlFunction?: ProxyConfigurationFunction;
17}
18/**
19 * The main purpose of the ProxyInfo object is to provide information
20 * about the current proxy connection used by the crawler for the request.
21 * Outside of crawlers, you can get this object by calling {@apilink ProxyConfiguration.newProxyInfo}.
22 *
23 * **Example usage:**
24 *
25 * ```javascript
26 * const proxyConfiguration = new ProxyConfiguration({
27 * proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
28 * });
29 *
30 * // Getting proxyInfo object by calling class method directly
31 * const proxyInfo = await proxyConfiguration.newProxyInfo();
32 *
33 * // In crawler
34 * const crawler = new CheerioCrawler({
35 * // ...
36 * proxyConfiguration,
37 * requestHandler({ proxyInfo }) {
38 * // Getting used proxy URL
39 * const proxyUrl = proxyInfo.url;
40 *
41 * // Getting ID of used Session
42 * const sessionIdentifier = proxyInfo.sessionId;
43 * }
44 * })
45 *
46 * ```
47 */
48export interface ProxyInfo {
49 /**
50 * The identifier of used {@apilink Session}, if used.
51 */
52 sessionId?: string;
53 /**
54 * The URL of the proxy.
55 */
56 url: string;
57 /**
58 * Username for the proxy.
59 */
60 username?: string;
61 /**
62 * User's password for the proxy.
63 */
64 password: string;
65 /**
66 * Hostname of your proxy.
67 */
68 hostname: string;
69 /**
70 * Proxy port.
71 */
72 port: number | string;
73}
74/**
75 * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
76 * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
77 * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
78 * the {@apilink ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
79 *
80 * If you want to use your own proxies, use the {@apilink ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
81 * be rotated by the configuration if this option is provided.
82 *
83 * **Example usage:**
84 *
85 * ```javascript
86 *
87 * const proxyConfiguration = new ProxyConfiguration({
88 * proxyUrls: ['...', '...'],
89 * });
90 *
91 * const crawler = new CheerioCrawler({
92 * // ...
93 * proxyConfiguration,
94 * requestHandler({ proxyInfo }) {
95 * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
96 * }
97 * })
98 *
99 * ```
100 * @category Scaling
101 */
102export declare class ProxyConfiguration {
103 isManInTheMiddle: boolean;
104 protected nextCustomUrlIndex: number;
105 protected proxyUrls?: string[];
106 protected usedProxyUrls: Map<string, string>;
107 protected newUrlFunction?: ProxyConfigurationFunction;
108 protected log: import("@apify/log").Log;
109 /**
110 * Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
111 * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
112 * them to use the selected proxies for all connections.
113 *
114 * ```javascript
115 * const proxyConfiguration = new ProxyConfiguration({
116 * proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
117 * });
118 *
119 * const crawler = new CheerioCrawler({
120 * // ...
121 * proxyConfiguration,
122 * requestHandler({ proxyInfo }) {
123 * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
124 * }
125 * })
126 *
127 * ```
128 */
129 constructor(options?: ProxyConfigurationOptions);
130 /**
131 * This function creates a new {@apilink ProxyInfo} info object.
132 * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
133 * the currently used proxy via the requestHandler parameter `proxyInfo`.
134 * Use it if you want to work with a rich representation of a proxy URL.
135 * If you need the URL string only, use {@apilink ProxyConfiguration.newUrl}.
136 * @param [sessionId]
137 * Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or
138 * you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
139 * When the provided sessionId is a number, it's converted to a string. Property sessionId of
140 * {@apilink ProxyInfo} is always returned as a type string.
141 *
142 * All the HTTP requests going through the proxy with the same session identifier
143 * will use the same target proxy server (i.e. the same IP address).
144 * The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
145 * @return Represents information about used proxy and its configuration.
146 */
147 newProxyInfo(sessionId?: string | number): Promise<ProxyInfo>;
148 /**
149 * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
150 * @param [sessionId]
151 * Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or
152 * you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
153 * When the provided sessionId is a number, it's converted to a string.
154 *
155 * All the HTTP requests going through the proxy with the same session identifier
156 * will use the same target proxy server (i.e. the same IP address).
157 * The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
158 * @return A string with a proxy URL, including authentication credentials and port number.
159 * For example, `http://bob:password123@proxy.example.com:8000`
160 */
161 newUrl(sessionId?: string | number): Promise<string>;
162 /**
163 * Handles custom url rotation with session
164 */
165 protected _handleCustomUrl(sessionId?: string): string;
166 /**
167 * Calls the custom newUrlFunction and checks format of its return value
168 */
169 protected _callNewUrlFunction(sessionId?: string): Promise<string>;
170 protected _throwNewUrlFunctionInvalid(err: Error): never;
171 protected _throwCannotCombineCustomMethods(): never;
172 protected _throwNoOptionsProvided(): never;
173}
174//# sourceMappingURL=proxy_configuration.d.ts.map
\No newline at end of file