interface RobotsChecker {
    isAllowed(url: string, userAgent?: string): boolean;
    getCrawlDelay(userAgent?: string): number | undefined;
}
export declare function getRobotsChecker(origin: string, userAgent?: string): Promise<RobotsChecker>;
export declare function isAllowedByRobots(url: string, userAgent?: string): Promise<boolean>;
export declare function getCrawlDelay(url: string, userAgent?: string): Promise<number>;
export {};