import utils import os.path import os import re import datetime import json import lxml.etree import uuid import logging import mechanize import zipfile import StringIO import requests import subprocess from email.utils import parsedate from time import mktime # to get text files their is a new dependency; you need to have pdftotext. # On Ubuntu, apt-get install poppler-utils. On OS X, install it via MacPorts # with port install poppler, or via Homebrew with brew install poppler. # options: # # --chamber: "house" or "senate" to limit the parse to a single chamber # --load_by: Takes a range of House Event IDs. Give it the beginning and end IDs with a dash between, otherwise, it goes by the committee feeds. # --docs=False: Don't download (& convert to text) House committee documents def run(options): # can limit it to one chamber chamber = options.get("chamber", None) if chamber and (chamber in ("house", "senate")): chambers = (chamber) else: chambers = ("house", "senate") load_by = options.get("load_by", None) # Load the committee metadata from the congress-legislators repository and make a # mapping from thomas_id and house_id to the committee dict. For each committee, # replace the subcommittees list with a dict from thomas_id to the subcommittee. utils.require_congress_legislators_repo() committees = {} for c in utils.yaml_load("congress-legislators/committees-current.yaml"): committees[c["thomas_id"]] = c if "house_committee_id" in c: committees[c["house_committee_id"] + "00"] = c c["subcommittees"] = dict((s["thomas_id"], s) for s in c.get("subcommittees", [])) if "senate" in chambers: print "Fetching Senate meetings..." meetings = fetch_senate_committee_meetings(committees, options) print "Writing Senate meeting data to disk." utils.write_json(meetings, output_for("senate")) if "house" in chambers: if load_by == None: print "Fetching House meetings..." meetings = fetch_house_committee_meetings(committees, options) else: print "Fetching House meetings by event_id..." meetings = fetch_meeting_from_event_id(committees, options, load_by) print "Writing House meeting data to disk." utils.write_json(meetings, output_for("house")) # Write all meetings to a single file on disk. # TODO: if these have unique IDs, maybe worth storing a file per-meeting. def output_for(chamber): return utils.data_dir() + "/committee_meetings_%s.json" % chamber # Parse the Senate committee meeting XML feed for meetings. # To aid users of the data, attempt to assign GUIDs to meetings. def fetch_senate_committee_meetings(committees, options): # Load any existing meetings file so we can recycle any GUIDs. existing_meetings = [] output_file = output_for("senate") if os.path.exists(output_file): existing_meetings = json.load(open(output_file)) options = dict(options) # clone options["binary"] = True # options["force"] = True meetings = [] dom = lxml.etree.fromstring(utils.download( "http://www.senate.gov/general/committee_schedules/hearings.xml", "committee_schedule/senate.xml", options)) for node in dom.xpath("meeting"): committee_id = unicode(node.xpath('string(cmte_code)')) if committee_id.strip() == "": continue # "No committee hearings scheduled" placeholder occurs_at = unicode(node.xpath('string(date)')) room = unicode(node.xpath('string(room)')) topic = unicode(node.xpath('string(matter)')) occurs_at = datetime.datetime.strptime(occurs_at, "%d-%b-%Y %I:%M %p") topic = re.sub(r"\s+", " ", topic).strip() # Validate committee code. try: committee_code, subcommittee_code = re.match(r"(\D+)(\d+)$", committee_id).groups() if committee_code not in committees: raise ValueError(committee_code) if subcommittee_code == "00": subcommittee_code = None if subcommittee_code and subcommittee_code not in committees[committee_code]["subcommittees"]: raise ValueError(subcommittee_code) except: print "Invalid committee code", committee_id continue # See if this meeting already exists. If so, take its GUID. # Assume meetings are the same if they are for the same committee/subcommittee and # at the same time. for mtg in existing_meetings: if mtg["committee"] == committee_code and mtg.get("subcommittee", None) == subcommittee_code and mtg["occurs_at"] == occurs_at.isoformat(): if options.get("debug", False): print "[%s] Reusing gUID." % mtg["guid"] guid = mtg["guid"] break else: # Not found, so create a new ID. # TODO: Can we make this a human-readable ID? guid = unicode(uuid.uuid4()) # Scrape the topic text for mentions of bill numbers. congress = utils.congress_from_legislative_year(utils.current_legislative_year(occurs_at)) bills = [] bill_number_re = re.compile(r"(hr|s|hconres|sconres|hjres|sjres|hres|sres)\s?(\d+)", re.I) for bill_match in bill_number_re.findall(topic.replace(".", "")): bills.append(bill_match[0].lower() + bill_match[1] + "-" + str(congress)) # Create the meeting event. if options.get("debug", False): print "[senate][%s][%s] Found meeting in room %s at %s." % (committee_code, subcommittee_code, room, occurs_at.isoformat()) meetings.append({ "chamber": "senate", "congress": congress, "guid": guid, "committee": committee_code, "subcommittee": subcommittee_code, "occurs_at": occurs_at.isoformat(), "room": room, "topic": topic, "bill_ids": bills, }) print "[senate] Found %i meetings." % len(meetings) return meetings # House # Scrape docs.house.gov for meetings. # To aid users of the data, assign GUIDs to meetings piggy-backing off of the provided EventID. def fetch_house_committee_meetings(committees, options): # Load any existing meetings file so we can recycle any GUIDs. existing_meetings = [] output_file = output_for("house") if os.path.exists(output_file): existing_meetings = json.load(open(output_file)) opts = dict(options) opts["binary"] = True opts["force"] = True meetings = [] seen_meetings = set() # Scrape the committee listing page for a list of committees with scrapable events. committee_html = utils.download("http://docs.house.gov/Committee/Committees.aspx", "committee_schedule/house_overview.html", options) for cmte in re.findall(r'