import utils import logging import re import json from lxml import etree import copy import datetime def create_govtrack_xml(bill, options): govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'} root = etree.Element("bill") root.set("session", bill['congress']) root.set("type", govtrack_type_codes[bill['bill_type']]) root.set("number", bill['number']) root.set("updated", utils.format_datetime(bill['updated_at'])) def make_node(parent, tag, text, **attrs): if options.get("govtrack", False): # Rewrite bioguide_id attributes as just id with GovTrack person IDs. attrs2 = {} for k, v in attrs.items(): if v: if k == "bioguide_id": # remap "bioguide_id" attributes to govtrack "id" k = "id" v = str(utils.translate_legislator_id('bioguide', v, 'govtrack')) attrs2[k] = v attrs = attrs2 return utils.make_node(parent, tag, text, **attrs) # for American Memory Century of Lawmaking bills... for source in bill.get("sources", []): n = make_node(root, "source", "") for k, v in sorted(source.items()): if k == "source": n.text = v elif k == "source_url": n.set("url", v) else: n.set(k, unicode(v)) if "original_bill_number" in bill: make_node(root, "bill-number", bill["original_bill_number"]) make_node(root, "state", bill['status'], datetime=bill['status_at']) old_status = make_node(root, "status", None) make_node(old_status, "introduced" if bill['status'] in ("INTRODUCED", "REFERRED") else "unknown", None, datetime=bill['status_at']) # dummy for the sake of comparison make_node(root, "introduced", None, datetime=bill['introduced_at']) titles = make_node(root, "titles", None) for title in bill['titles']: n = make_node(titles, "title", title['title']) n.set("type", title['type']) if title['as']: n.set("as", title['as']) if title['is_for_portion']: n.set("partial", "1") if bill['sponsor']: # TODO: Sponsored by committee? make_node(root, "sponsor", None, bioguide_id=bill['sponsor']['bioguide_id']) else: make_node(root, "sponsor", None) cosponsors = make_node(root, "cosponsors", None) for cosp in bill['cosponsors']: n = make_node(cosponsors, "cosponsor", None, bioguide_id=cosp["bioguide_id"]) if cosp["sponsored_at"]: n.set("joined", cosp["sponsored_at"]) if cosp["withdrawn_at"]: n.set("withdrawn", cosp["withdrawn_at"]) actions = make_node(root, "actions", None) for action in bill['actions']: a = make_node(actions, action['type'] if action['type'] in ("vote", "vote-aux", "calendar", "topresident", "signed", "enacted", "vetoed") else "action", None, datetime=action['acted_at']) if action.get("status"): a.set("state", action["status"]) if action['type'] in ('vote', 'vote-aux'): a.clear() # re-insert date between some of these attributes a.set("how", action["how"]) a.set("type", action["vote_type"]) if action.get("roll") != None: a.set("roll", action["roll"]) a.set("datetime", utils.format_datetime(action['acted_at'])) a.set("where", action["where"]) a.set("result", action["result"]) if action.get("suspension"): a.set("suspension", "1") if action.get("status"): a.set("state", action["status"]) if action['type'] == 'calendar' and "calendar" in action: a.set("calendar", action["calendar"]) if action["under"]: a.set("under", action["under"]) if action["number"]: a.set("number", action["number"]) if action['type'] == 'enacted': a.clear() # re-insert date between some of these attributes a.set("number", "%s-%s" % (bill['congress'], action["number"])) a.set("type", action["law"]) a.set("datetime", utils.format_datetime(action['acted_at'])) if action.get("status"): a.set("state", action["status"]) if action['type'] == 'vetoed': if action.get("pocket"): a.set("pocket", "1") if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) committees = make_node(root, "committees", None) for cmt in bill['committees']: make_node(committees, "committee", None, code=(cmt["committee_id"] + cmt["subcommittee_id"]) if cmt.get("subcommittee_id", None) else cmt["committee_id"], name=cmt["committee"], subcommittee=cmt.get("subcommittee").replace("Subcommittee on ", "") if cmt.get("subcommittee") else "", activity=", ".join(c.title() for c in cmt["activity"])) relatedbills = make_node(root, "relatedbills", None) for rb in bill['related_bills']: if rb['type'] == "bill": rb_bill_type, rb_number, rb_congress = utils.split_bill_id(rb['bill_id']) make_node(relatedbills, "bill", None, session=rb_congress, type=govtrack_type_codes[rb_bill_type], number=rb_number, relation="unknown" if rb['reason'] == "related" else rb['reason']) subjects = make_node(root, "subjects", None) if bill['subjects_top_term']: make_node(subjects, "term", None, name=bill['subjects_top_term']) for s in bill['subjects']: if s != bill['subjects_top_term']: make_node(subjects, "term", None, name=s) amendments = make_node(root, "amendments", None) for amd in bill['amendments']: make_node(amendments, "amendment", None, number=amd["chamber"] + str(amd["number"])) if bill.get('summary'): make_node(root, "summary", bill['summary']['text'], date=bill['summary']['date'], status=bill['summary']['as']) committee_reports = make_node(root, "committee-reports", None) for report in bill['committee_reports']: make_node(committee_reports, "report", report) return etree.tostring(root, pretty_print=True) def sponsor_for(sponsor_dict): if sponsor_dict is None: # TODO: This can hopefully be removed. In testing s414-113 # was missing sponsor data. But all bills have a sponsor? return None # TODO: Don't do regex matching here. Find another way. m = re.match(r'(?P(Rep|Sen))\. (?P<name>.*?) +\[(?P<party>[DRI])-(?P<state>[A-Z][A-Z])(-(?P<district>\d{1,2}|At Large))?\]$', sponsor_dict['fullName']) if m.group("district") is None: district = None # a senator elif m.group("district") == "At Large": district = None # TODO: For backwards compatibility, we're returning None, but 0 would be better. else: # TODO: For backwards compatibility, we're returning a string, but an int would be better. district = m.group('district') return { 'title': m.group("title"), 'name': m.group("name"), # the firstName, middleName, lastName fields have inconsistent capitalization - some are all uppercase 'district': district, 'state': m.group('state'), #'party': m.group('party'), 'bioguide_id': sponsor_dict['bioguideId'], 'type': 'person' } def summary_for(summaries): # Some bills are missing the summaries entirely? if summaries is None: return None # Take the most recent summary, by looking at the lexicographically last updateDate. summaries = summaries['item'] summary = sorted(summaries, key = lambda s: s['updateDate'])[-1] # Build dict. return { "date": summary['updateDate'], "as": summary['name'], "text": strip_tags(summary['text']), } def strip_tags(text): # Preserve paragraph breaks. Convert closing p tags (and surrounding whitespace) into two newlines. Strip trailing whitespace text = re.sub("\s*</\s*p\s*>\s*", "\n\n", text).strip() # naive stripping of tags, should work okay in this limited context text = re.sub("<[^>]+>", "", text) # compress and strip whitespace artifacts, except for the paragraph breaks text = re.sub("[ \t\r\f\v]{2,}", " ", text).strip() # Replace HTML entities with characters. text = utils.unescape(text) return text def committees_for(committee_list): if committee_list is None: return [] committee_list = committee_list['item'] activity_text_map = { "Referred to": ["referral"], "Hearings by": ["hearings"], "Markup by": ["markup"], "Reported by": ["reporting"], "Discharged from": ["discharged"], "Reported original measure": ["origin", "reporting"], } def fix_subcommittee_name(name): return re.sub("(.*) Subcommittee$", lambda m : "Subcommittee on " + m.group(1), name) def get_activitiy_list(item): if not item['activities']: return [] return sum([activity_text_map.get(i['name'], [i['name']]) for i in item['activities']['item']], []) def fixup_committee_name(name): # Preserve backwards compatiblity. if name == "House House Administration": return "House Administration" return name def build_dict(item): committee_dict = { 'activity': get_activitiy_list(item), 'committee': fixup_committee_name(item['chamber'] + ' ' + re.sub(" Committee$", "", item['name'])), 'committee_id': item['systemCode'][0:-2].upper(), } subcommittees_list = [] if 'subcommittees' in item and item['subcommittees'] is not None: for subcommittee in item['subcommittees']['item']: subcommittee_dict = copy.deepcopy(committee_dict) subcommittee_dict.update({ 'subcommittee': fix_subcommittee_name(subcommittee['name']), 'subcommittee_id': subcommittee['systemCode'][-2:], 'activity': get_activitiy_list(subcommittee), }) subcommittees_list.append(subcommittee_dict) return [committee_dict] + subcommittees_list return sum([build_dict(committee) for committee in committee_list], []) def titles_for(title_list): def build_dict(item): full_type = item['titleType'] is_for_portion = False # "Official Titles as Introduced", "Short Titles on Conference report" splits = re.split(" as | on ", full_type, 1) if len(splits) == 2: title_type, state = splits if state.endswith(" for portions of this bill"): is_for_portion = True state = state.replace(" for portions of this bill" ,"") state = state.replace(":", "").lower() else: title_type, state = full_type, None if "Popular Title" in title_type: title_type = "popular" elif "Short Title" in title_type: title_type = "short" elif "Official Title" in title_type: title_type = "official" elif "Display Title" in title_type: title_type = "display" elif title_type == "Non-bill-report": # TODO: What kind of title is this? Maybe assign # a better title_type code once we know. title_type = "nonbillreport" else: raise Exception("Unknown title type: " + title_type) return { 'title': item['title'], 'is_for_portion': is_for_portion, 'as': state, 'type': title_type } titles = [build_dict(title) for title in title_list] # THOMAS used to give us the titles in a particular order: # short as introduced # short as introduced (for portion) # short as some later stage # short as some later stage (for portion) # official as introduced # official as some later stage # The "as" stages (introduced, etc.) were in the order in which actions # actually occurred. This was handy because to get the current title for # a bill, you need to know which action type was most recent. The new # order is reverse-chronological, so we have to turn the order around # for backwards compatibility. Rather than do a simple .reverse(), I'm # adding an explicit sort order here which gets very close to the THOMAS # order. # Unfortunately this can no longer be relied on because the new bulk # data has the "as" stages sometimes in the wrong order: The "reported to # senate" status for House bills seems to be consistently out of place. titles_copy = list(titles) # clone before beginning sort def first_index_of(**kwargs): for i, title in enumerate(titles_copy): for k, v in kwargs.items(): k = k.replace("_", "") if title.get(k) != v: break else: # break not called --- all match return i titles.sort(key = lambda title: ( # keep the same 'short', 'official', 'display' order intact first_index_of(type=title['type']), # within each of those categories, reverse the 'as' order -first_index_of(type=title['type'], _as=title.get('as')), # put titles for portions last, within the type/as category title['is_for_portion'], # and within that, just sort alphabetically, case-insensitively (which is # what it appears THOMAS used to do) title['title'].lower(), )) return titles # the most current title of a given type is the first one in the last 'as' subgroup # of the titles for the whole bill (that is, if there's no title for the whole bill # in the last 'as' subgroup, use the previous 'as' subgroup and so on) --- we think # this logic matches THOMAS/Congress.gov. def current_title_for(titles, title_type): current_title = None current_as = -1 # not None, cause for popular titles, None is a valid 'as' for title in titles: if title['type'] != title_type or title['is_for_portion'] == True: continue if title['as'] == current_as: continue # right type, new 'as', store first one current_title = title['title'] current_as = title['as'] return current_title def actions_for(action_list, bill_id, title): # The bulk XML data has action history information from multiple sources. For # major actions, the Library of Congress (code 9) action item often duplicates # the information of a House/Senate action item. We have to skip one so that we # don't tag multiple history items with the same parsed action info, which # would imply the action (like a vote) ocurred multiple times. THOMAS appears # to have suppressed the Library of Congress action lines in certain cases # to avoid duplication - they were not in our older data files. # # Also, there are some ghost action items with totally empty text. Remove those. # TODO: When removed from upstream data, we can remove that check. closure = { "prev": None, } def keep_action(item, closure): if item['text'] in (None, ""): return False keep = True if closure['prev']: if item['sourceSystem']['code'] == "9": # Date must match previous action.. # If both this and previous have a time, the times must match. # The text must approximately match. Sometimes the LOC text has a prefix # and different whitespace. And they may drop references -- so we'll # use our action_for helper function to drop references from both # prior to the string comparison. if item['actionDate'] == closure["prev"]["actionDate"] \ and (item.get('actionTime') == closure["prev"].get("actionTime") or not item.get('actionTime') or not closure["prev"].get("actionTime")) \ and action_for(item)['text'].replace(" ", "").endswith(action_for(closure["prev"])['text'].replace(" ", "")): keep = False closure['prev'] = item return keep action_list = [item for item in action_list if keep_action(item, closure)] # Turn the actions into dicts. The actions are in reverse-chronological # order in the bulk data XML. Process them in chronological order so that # our bill status logic sees the actions in the right order. def build_dict(item, closure): action_dict = action_for(item) extra_action_info, new_status = parse_bill_action(action_dict, closure['prev_status'], bill_id, title) # only change/reflect status change if there was one if new_status: action_dict['status'] = new_status closure['prev_status'] = new_status # add additional parsed fields if extra_action_info: action_dict.update(extra_action_info) return action_dict closure = { "prev_status": "INTRODUCED", } return [build_dict(action, closure) for action in reversed(action_list)] # clean text, pull out the action type, any other associated metadata with an action def action_for(item): # acted_at if not item.get('actionTime'): acted_at = item.get('actionDate', '') else: # Although we get the action date & time in an ISO-ish format (split # across two fields), and although we know it's in local time at the # U.S. Capitol (i.e. U.S. Eastern), we don't know the UTC offset which # is a part of how we used to serialize the time. So parse and then # use pytz (via format_datetime) to re-serialize. acted_at = utils.format_datetime(datetime.datetime.strptime(item.get('actionDate', '') + " " + item['actionTime'], "%Y-%m-%d %H:%M:%S")) # text & references # (amendment actions don't always have text?) text = item['text'] if item['text'] is not None else '' # strip out links text = re.sub(r"</?[Aa]( \S.*?)?>", "", text) # remove and extract references references = [] match = re.search("\s*\(([^)]+)\)\s*$", text) if match: # remove the matched section text = text[0:match.start()] + text[match.end():] types = match.group(1) # fix use of comma or colon instead of a semi colon between reference types # have seen some accidental capitalization combined with accidental comma, thus the 'T' # e.g. "text of Title VII as reported in House: CR H3075-3077, Text omission from Title VII:" (hr5384-109) types = re.sub("[,:] ([a-zT])", r"; \1", types) # fix "CR:" types = re.sub("CR:", "CR", types) # fix a missing semicolon altogether between references # e.g. sres107-112, "consideration: CR S1877-1878 text as" types = re.sub("(\d+) +([a-z])", r"\1; \2", types) for reference in re.split("; ?", types): if ": " not in reference: type, reference = None, reference else: type, reference = reference.split(": ", 1) references.append({'type': type, 'reference': reference}) # form dict action_dict = { 'acted_at': acted_at, 'action_code': item.get('actionCode', ''), 'committees': [item['committee']['systemCode'][0:-2].upper()] if item['committee'] else None, 'references': references, 'type': 'action', # replaced by parse_bill_action if a regex matches 'text': text, } if not action_dict["committees"]: # remove if empty - not present in how we used to generate the file del action_dict["committees"] return action_dict def cosponsors_for(cosponsors_list): if cosponsors_list is None: return [] cosponsors_list = cosponsors_list['item'] def build_dict(item): cosponsor_dict = sponsor_for(item) del cosponsor_dict["type"] # always 'person' cosponsor_dict.update({ 'sponsored_at': item['sponsorshipDate'], 'withdrawn_at': item['sponsorshipWithdrawnDate'], 'original_cosponsor': item['isOriginalCosponsor'] == 'True' }) return cosponsor_dict cosponsors = [build_dict(cosponsor) for cosponsor in cosponsors_list] # TODO: Can remove. Sort like the old THOMAS order to make diffs easier. cosponsors.sort(key = lambda c: c['name'].lower()) return cosponsors def related_bills_for(related_bills_list): if related_bills_list is None: return [] related_bills_list = related_bills_list['item'] def build_dict(item): return { 'reason': item['relationshipDetails']['item'][0]['type'].replace('bill', '').strip().lower(), 'bill_id': '{0}{1}-{2}'.format(item['type'].replace('.', '').lower(), item['number'], item['congress']), 'type': 'bill', 'identified_by': item['relationshipDetails']['item'][0]['identifiedBy'] } # Are these THOMAS related bill relation texts gone from the bulk data? reasons = ( ("Identical bill identified by (CRS|House|Senate)", "identical"), ("Companion bill", "identical"), ("Related bill (as )?identified by (CRS|the House Clerk's office|House committee|Senate)", "related"), ("passed in (House|Senate) in lieu of .*", "supersedes"), ("Rule related to .* in (House|Senate)", "rule"), ("This bill has text inserted from .*", "includes"), ("Text from this bill was inserted in .*", "included-in"), ("Bill related to rule .* in House", "ruled-by"), ("This bill caused other related action on .*", "caused-action"), ("Other related action happened to this bill because of .*", "action-caused-by"), ("Bill that causes .* to be laid on table in House", "caused-action"), ("Bill laid on table by virtue of .* passage in House", "action-caused-by"), ("Bill that caused the virtual passage of .* in House", "caused-action"), ("Bill passed by virtue of .* passage in House", "caused-action-by"), ("Bill on wich enrollment has been corrected by virtue of .* passage in House", "caused-action"), ) return [build_dict(related_bill) for related_bill in related_bills_list] # get the public or private law number from any enacted action def slip_law_from(actions): for action in actions: if action["type"] == "enacted": return { 'law_type': action["law"], 'congress': action["congress"], 'number': action["number"] } # find the latest status change in a set of processed actions def latest_status(actions, introduced_at): status, status_date = "INTRODUCED", introduced_at for action in actions: if action.get('status', None): status = action['status'] status_date = action['acted_at'] return status, status_date # look at the final set of processed actions and pull out the major historical events def history_from_actions(actions): history = {} activation = activation_from(actions) if activation: history['active'] = True history['active_at'] = activation['acted_at'] else: history['active'] = False house_vote = None for action in actions: if (action['type'] == 'vote') and (action['where'] == 'h') and (action['vote_type'] != "override"): house_vote = action if house_vote: history['house_passage_result'] = house_vote['result'] history['house_passage_result_at'] = house_vote['acted_at'] senate_vote = None for action in actions: if (action['type'] == 'vote') and (action['where'] == 's') and (action['vote_type'] != "override"): senate_vote = action if senate_vote: history['senate_passage_result'] = senate_vote['result'] history['senate_passage_result_at'] = senate_vote['acted_at'] senate_vote = None for action in actions: if (action['type'] == 'vote-aux') and (action['vote_type'] == 'cloture') and (action['where'] == 's') and (action['vote_type'] != "override"): senate_vote = action if senate_vote: history['senate_cloture_result'] = senate_vote['result'] history['senate_cloture_result_at'] = senate_vote['acted_at'] vetoed = None for action in actions: if action['type'] == 'vetoed': vetoed = action if vetoed: history['vetoed'] = True history['vetoed_at'] = vetoed['acted_at'] else: history['vetoed'] = False house_override_vote = None for action in actions: if (action['type'] == 'vote') and (action['where'] == 'h') and (action['vote_type'] == "override"): house_override_vote = action if house_override_vote: history['house_override_result'] = house_override_vote['result'] history['house_override_result_at'] = house_override_vote['acted_at'] senate_override_vote = None for action in actions: if (action['type'] == 'vote') and (action['where'] == 's') and (action['vote_type'] == "override"): senate_override_vote = action if senate_override_vote: history['senate_override_result'] = senate_override_vote['result'] history['senate_override_result_at'] = senate_override_vote['acted_at'] enacted = None for action in actions: if action['type'] == 'enacted': enacted = action if enacted: history['enacted'] = True history['enacted_at'] = action['acted_at'] else: history['enacted'] = False topresident = None for action in actions: if action['type'] == 'topresident': topresident = action if topresident and (not history['vetoed']) and (not history['enacted']): history['awaiting_signature'] = True history['awaiting_signature_since'] = action['acted_at'] else: history['awaiting_signature'] = False return history # find the first action beyond the standard actions every bill gets. # - if the bill's first action is "referral" then the first action not those # most common # e.g. hr3590-111 (active), s1-113 (inactive) # - if the bill's first action is "action", then the next action, if one is present # resolutions # e.g. sres5-113 (active), sres4-113 (inactive) # - if the bill's first action is anything else (e.g. "vote"), then that first action # bills that skip committee # e.g. s227-113 (active) def activation_from(actions): # there's NOT always at least one :( # as of 2013-06-10, hr2272-113 has no actions at all if len(actions) == 0: return None first = actions[0] if first['type'] in ["referral", "calendar", "action"]: for action in actions[1:]: if (action['type'] != "referral") and (action['type'] != "calendar") and ("Sponsor introductory remarks" not in action['text']): return action return None else: return first def parse_bill_action(action_dict, prev_status, bill_id, title): """Parse a THOMAS bill action line. Returns attributes to be set in the XML file on the action line.""" bill_type, number, congress = utils.split_bill_id(bill_id) line = action_dict['text'] status = None action = { "type": "action" } # If a line starts with an amendment number, this action is on the amendment and cannot # be parsed yet. m = re.search(r"^(H|S)\.Amdt\.(\d+)", line, re.I) if m != None: # Process actions specific to amendments separately. return None, None # Otherwise, parse the action line for key actions. # VOTES # A House Vote. line = re.sub(", the Passed", ", Passed", line) # 106 h4733 and others m = re.search("(" + "|".join([ "On passage", "Passed House", "Two-thirds of the Members present having voted in the affirmative the bill is passed,?", "On motion to suspend the rules and pass the (?:bill|resolution)", "On agreeing to the (?:resolution|conference report)", "On motion to suspend the rules and agree to the (?:resolution|conference report)", "House Agreed to Senate Amendments.*?", "On motion that the House (?:suspend the rules and )?(?:agree(?: with an amendment)? to|concur in) the Senate amendments?(?: to the House amendments?| to the Senate amendments?)*", ]) + ")" + "(, the objections of the President to the contrary notwithstanding.?)?" + "(, as amended| \(Amended\))?" + " (Passed|Failed|Agreed to|Rejected)?" + " ?(by voice vote|without objection|by (the Yeas and Nays|Yea-Nay Vote|recorded vote)" + "(:? \(2/3 required\))?: (\d+ - \d+(, \d+ Present)? [ \)]*)?\((Roll no\.|Record Vote No:) \d+\))", line, re.I) if m != None: motion, is_override, as_amended, pass_fail, how = m.group(1), m.group(2), m.group(3), m.group(4), m.group(5) # print line # print m.groups() if re.search(r"Passed House|House Agreed to", motion, re.I): pass_fail = 'pass' elif re.search("(ayes|yeas) had prevailed", line, re.I): pass_fail = 'pass' elif re.search(r"Pass|Agreed", pass_fail, re.I): pass_fail = 'pass' else: pass_fail = 'fail' if "Two-thirds of the Members present" in motion: is_override = True if is_override: vote_type = "override" elif re.search(r"(agree (with an amendment )?to|concur in) the Senate amendment", line, re.I): vote_type = "pingpong" elif re.search("conference report", line, re.I): vote_type = "conference" elif bill_type[0] == "h": vote_type = "vote" else: vote_type = "vote2" roll = None m = re.search(r"\((Roll no\.|Record Vote No:) (\d+)\)", how, re.I) if m != None: how = "roll" # normalize the ugly how roll = m.group(2) suspension = None if roll and "On motion to suspend the rules" in motion: suspension = True # alternate form of as amended, e.g. hr3979-113 if "that the House agree with an amendment" in motion: as_amended = True action["type"] = "vote" action["vote_type"] = vote_type action["how"] = how action['where'] = "h" action['result'] = pass_fail if roll: action["roll"] = roll action["suspension"] = suspension # correct upstream data error if bill_id == "s2012-114" and "Roll no. 250" in line: as_amended = True if bill_id == "s2943-114" and "On passage Passed without objection" in line: as_amended = True # get the new status of the bill after this vote new_status = new_status_after_vote(vote_type, pass_fail == "pass", "h", bill_type, suspension, as_amended, title, prev_status) if new_status: status = new_status # Passed House, not necessarily by an actual vote (think "deem") m = re.search(r"Passed House pursuant to|House agreed to Senate amendment (with amendment )?pursuant to", line, re.I) if m != None: vote_type = "vote" if (bill_type[0] == "h") else "vote2" if "agreed to Senate amendment" in line: vote_type = "pingpong" pass_fail = "pass" as_amended = bool(m.group(1)) action["type"] = "vote" action["vote_type"] = vote_type action["how"] = "by special rule" action["where"] = "h" action["result"] = pass_fail # get the new status of the bill after this vote new_status = new_status_after_vote(vote_type, pass_fail == "pass", "h", bill_type, False, as_amended, title, prev_status) if new_status: status = new_status # A Senate Vote # (There are some annoying weird cases of double spaces which are taken care of # at the end.) m = re.search("(" + "|".join([ "Passed Senate", "Failed of passage in Senate", "Disagreed to in Senate", "Resolution agreed to in Senate", "Senate (?:agreed to|concurred in) (?:the )?(?:conference report|House amendment(?: to the Senate amendments?| to the House amendments?)*)", r"Cloture \S*\s?on the motion to proceed .*?not invoked in Senate", r"Cloture(?: motion)? on the motion to proceed to the (?:bill|measure) invoked in Senate", "Cloture invoked in Senate", "Cloture on (?:the motion to proceed to )?the bill (?:not )?invoked in Senate", "(?:Introduced|Received|Submitted) in the Senate, (?:read twice, |considered, |read the third time, )+and (?:passed|agreed to)", ]) + ")" + "(,?.*,?) " + "(without objection|by Unanimous Consent|by Voice Vote|(?:by )?Yea-Nay( Vote)?\. \d+\s*-\s*\d+\. Record Vote (No|Number): \d+)", line.replace(" ", " "), re.I) if m != None: motion, extra, how = m.group(1), m.group(2), m.group(3) roll = None # put disagreed check first, cause "agreed" is contained inside it if re.search("disagreed", motion, re.I): pass_fail = "fail" elif re.search("passed|agreed|concurred|bill invoked|measure invoked|cloture invoked", motion, re.I): pass_fail = "pass" else: pass_fail = "fail" voteaction_type = "vote" if re.search("over veto", extra, re.I): vote_type = "override" elif re.search("conference report", motion, re.I): vote_type = "conference" elif re.search("cloture", motion, re.I): vote_type = "cloture" voteaction_type = "vote-aux" # because it is not a vote on passage elif re.search("Senate agreed to (the )?House amendment|Senate concurred in (the )?House amendment", motion, re.I): vote_type = "pingpong" elif bill_type[0] == "s": vote_type = "vote" else: vote_type = "vote2" m = re.search(r"Record Vote (No|Number): (\d+)", how, re.I) if m != None: roll = m.group(2) how = "roll" as_amended = False if re.search(r"with amendments|with an amendment", extra, re.I): as_amended = True action["type"] = voteaction_type action["vote_type"] = vote_type action["how"] = how action["result"] = pass_fail action["where"] = "s" if roll: action["roll"] = roll # get the new status of the bill after this vote new_status = new_status_after_vote(vote_type, pass_fail == "pass", "s", bill_type, False, as_amended, title, prev_status) if new_status: status = new_status # OLD-STYLE VOTES (93rd Congress-ish) m = re.search(r"Measure passed (House|Senate)(, amended(?: \(.*?\)|, with an amendment to the title)?)?(?:,? in lieu[^,]*)?(?:, roll call #(\d+) \(\d+-\d+\))?", line, re.I) if m != None: chamber = m.group(1)[0].lower() # 'h' or 's' as_amended = m.group(2) roll_num = m.group(3) # GovTrack legacy scraper missed these: if chamber == 's' and (as_amended or roll_num or "lieu" in line): return action, status pass_fail = "pass" vote_type = "vote" if bill_type[0] == chamber else "vote2" action["type"] = "vote" action["vote_type"] = vote_type action["how"] = "(method not recorded)" if not roll_num else "roll" if roll_num: action["roll"] = roll_num action["result"] = pass_fail action["where"] = chamber new_status = new_status_after_vote(vote_type, pass_fail == "pass", chamber, bill_type, False, as_amended, title, prev_status) if new_status: status = new_status m = re.search(r"(House|Senate) agreed to (?:House|Senate) amendments?( with an amendment)?( under Suspension of the Rules)?(?:, roll call #(\d+) \(\d+-\d+\))?\.", line, re.I) if m != None: chamber = m.group(1)[0].lower() # 'h' or 's' as_amended = m.group(2) suspension = m.group(3) roll_num = m.group(4) # GovTrack legacy scraper missed these: if (chamber == 'h' and not roll_num) or (chamber == 's' and rull_num): return action, status # REMOVE ME pass_fail = "pass" vote_type = "pingpong" action["type"] = "vote" action["vote_type"] = vote_type action["how"] = "(method not recorded)" if not roll_num else "roll" if roll_num: action["roll"] = roll_num action["result"] = pass_fail action["where"] = chamber action["suspension"] = (suspension != None) new_status = new_status_after_vote(vote_type, pass_fail == "pass", chamber, bill_type, False, as_amended, title, prev_status) if new_status: status = new_status # PSUDO-REPORTING (because GovTrack did this, but should be changed) # TODO: Make a new status for this as pre-reported. m = re.search(r"Placed on (the )?([\w ]+) Calendar( under ([\w ]+))?[,\.] Calendar No\. (\d+)\.|Committee Agreed to Seek Consideration Under Suspension of the Rules|Ordered to be Reported", line, re.I) if m != None: # TODO: This makes no sense. if prev_status in ("INTRODUCED", "REFERRED"): status = "REPORTED" action["type"] = "calendar" # TODO: Useless. But good for GovTrack compatibility. if m.group(2): # not 'Ordered to be Reported' action["calendar"] = m.group(2) action["under"] = m.group(4) action["number"] = m.group(5) # COMMITTEE ACTIONS # reported m = re.search(r"Committee on (.*)\. Reported by", line, re.I) if m != None: action["type"] = "reported" action["committee"] = m.group(1) if prev_status in ("INTRODUCED", "REFERRED"): status = "REPORTED" m = re.search(r"Reported to Senate from the (.*?)( \(without written report\))?\.", line, re.I) if m != None: # 93rd Congress action["type"] = "reported" action["committee"] = m.group(1) if prev_status in ("INTRODUCED", "REFERRED"): status = "REPORTED" # hearings held by a committee m = re.search(r"(Committee on .*?)\. Hearings held", line, re.I) if m != None: action["committee"] = m.group(1) action["type"] = "hearings" m = re.search(r"Committee on (.*)\. Discharged (by Unanimous Consent)?", line, re.I) if m != None: action["committee"] = m.group(1) action["type"] = "discharged" if prev_status in ("INTRODUCED", "REFERRED"): status = "REPORTED" m = re.search("Cleared for White House|Presented to President", line, re.I) if m != None: action["type"] = "topresident" m = re.search("Signed by President", line, re.I) if m != None: action["type"] = "signed" status = "ENACTED:SIGNED" m = re.search("Pocket Vetoed by President", line, re.I) if m != None: action["type"] = "vetoed" action["pocket"] = "1" status = "VETOED:POCKET" # need to put this in an else, or this regex will match the pocket veto and override it else: m = re.search("Vetoed by President", line, re.I) if m != None: action["type"] = "vetoed" status = "PROV_KILL:VETO" m = re.search("Sent to Archivist of the United States unsigned", line, re.I) if m != None: status = "ENACTED:TENDAYRULE" m = re.search("^(?:Became )?(Public|Private) Law(?: No:)? ([\d\-]+)\.", line, re.I) if m != None: action["law"] = m.group(1).lower() pieces = m.group(2).split("-") action["congress"] = pieces[0] action["number"] = pieces[1] action["type"] = "enacted" if prev_status in ("ENACTED:SIGNED", "ENACTED:VETO_OVERRIDE", "ENACTED:TENDAYRULE"): pass # this is a final administrative step elif prev_status == "PROV_KILL:VETO" or prev_status.startswith("VETOED:"): # somehow missed the override steps status = "ENACTED:VETO_OVERRIDE" elif bill_id in ("s2641-93", "hr1589-94", "s2527-100", "hr1677-101", "hr2978-101", "hr2126-104", "s1322-104"): status = "ENACTED:TENDAYRULE" else: raise Exception("Missing Signed by President action? If this is a case of the 10-day rule, hard code the bill id %s here." % bill_id) # Check for referral type m = re.search(r"Referred to (?:the )?(House|Senate)?\s?(?:Committee|Subcommittee)?", line, re.I) if m != None: action["type"] = "referral" if prev_status == "INTRODUCED": status = "REFERRED" # sweep the action line for bill IDs of related bills bill_ids = utils.extract_bills(line, congress) bill_ids = filter(lambda b: b != bill_id, bill_ids) if bill_ids and (len(bill_ids) > 0): action['bill_ids'] = bill_ids return action, status def new_status_after_vote(vote_type, passed, chamber, bill_type, suspension, amended, title, prev_status): if vote_type == "vote": # vote in originating chamber if passed: if bill_type in ("hres", "sres"): return 'PASSED:SIMPLERES' # end of life for a simple resolution if chamber == "h": return 'PASS_OVER:HOUSE' # passed by originating chamber, now in second chamber else: return 'PASS_OVER:SENATE' # passed by originating chamber, now in second chamber if suspension: return 'PROV_KILL:SUSPENSIONFAILED' # provisionally killed by failure to pass under suspension of the rules if chamber == "h": return 'FAIL:ORIGINATING:HOUSE' # outright failure else: return 'FAIL:ORIGINATING:SENATE' # outright failure if vote_type in ("vote2", "pingpong"): # vote in second chamber or subsequent pingpong votes if passed: if amended: # mesure is passed but not in identical form if chamber == "h": return 'PASS_BACK:HOUSE' # passed both chambers, but House sends it back to Senate else: return 'PASS_BACK:SENATE' # passed both chambers, but Senate sends it back to House else: # bills and joint resolutions not constitutional amendments, not amended from Senate version if bill_type in ("hjres", "sjres") and title.startswith("Proposing an amendment to the Constitution of the United States"): return 'PASSED:CONSTAMEND' # joint resolution that looks like an amendment to the constitution if bill_type in ("hconres", "sconres"): return 'PASSED:CONCURRENTRES' # end of life for concurrent resolutions return 'PASSED:BILL' # passed by second chamber, now on to president if vote_type == "pingpong": # chamber failed to accept the other chamber's changes, but it can vote again return 'PROV_KILL:PINGPONGFAIL' if suspension: return 'PROV_KILL:SUSPENSIONFAILED' # provisionally killed by failure to pass under suspension of the rules if chamber == "h": return 'FAIL:SECOND:HOUSE' # outright failure else: return 'FAIL:SECOND:SENATE' # outright failure if vote_type == "cloture": if not passed: return "PROV_KILL:CLOTUREFAILED" else: return None if vote_type == "override": if not passed: if bill_type[0] == chamber: if chamber == "h": return 'VETOED:OVERRIDE_FAIL_ORIGINATING:HOUSE' else: return 'VETOED:OVERRIDE_FAIL_ORIGINATING:SENATE' else: if chamber == "h": return 'VETOED:OVERRIDE_FAIL_SECOND:HOUSE' else: return 'VETOED:OVERRIDE_FAIL_SECOND:SENATE' else: if bill_type[0] == chamber: if chamber == "h": return 'VETOED:OVERRIDE_PASS_OVER:HOUSE' else: return 'VETOED:OVERRIDE_PASS_OVER:SENATE' else: # The override passed both chambers -- the veto is overridden. return "ENACTED:VETO_OVERRIDE" if vote_type == "conference": # This is tricky to integrate into status because we have to wait for both # chambers to pass the conference report. if passed: if prev_status.startswith("CONFERENCE:PASSED:"): if bill_type in ("hjres", "sjres") and title.startswith("Proposing an amendment to the Constitution of the United States"): return 'PASSED:CONSTAMEND' # joint resolution that looks like an amendment to the constitution if bill_type in ("hconres", "sconres"): return 'PASSED:CONCURRENTRES' # end of life for concurrent resolutions return 'PASSED:BILL' else: if chamber == "h": return 'CONFERENCE:PASSED:HOUSE' else: return 'CONFERENCE:PASSED:SENATE' return None def amendments_for(amendment_list): if amendment_list is None: return [] amendment_list = amendment_list['amendment'] def build_dict(item): # Malformed XML containing duplicate elements causes attributes to parse as a list for attr in ['type', 'number', 'congress']: if type(item[attr]) is list: item[attr] = item[attr][0] return { 'amendment_id': "{0}{1}-{2}".format(item['type'].lower(), item['number'], item['congress']), 'amendment_type': item['type'].lower(), 'chamber': item['type'][0].lower(), 'number': item['number'] } return [build_dict(amendment) for amendment in amendment_list] def committee_reports_for(committeeReports): ret = [] for report in (committeeReports or {}).get("committeeReport", []): ret.append( report["citation"] ) return ret