ensure all strings that look like integers are quoted (only needed for zero-lead octal-integer-like strings with '8' or '9' in the value which would previously omit quotes)

committees-current.yaml

@@ -150,7 +150,7 @@
     address: 2129 RHOB; Washington, DC 20515
     phone: (202) 225-7502
   - name: Oversight and Investigations
-    thomas_id: 09
+    thomas_id: '09'
     address: 2129 RHOB; Washington, DC 20515
     phone: (202) 225-7502
   address: 2129 RHOB; Washington, DC 20515-6050
@@ -270,7 +270,7 @@
     address: 1309 LHOB; Washington, DC 20515
     phone: (202) 225-8281
   - name: Elections
-    thomas_id: 08
+    thomas_id: '08'
     address: 1309 LHOB; Washington, DC 20515-6157
     phone: (202) 225-8281
   address: 1309 LHOB; Washington, DC 20515-6157
@@ -290,7 +290,7 @@
     address: H2-176 FHOB; Washington, DC 20515-6480
     phone: (202) 226-8417
   - name: Cybersecurity, Infrastructure Protection, and Security Technologies
-    thomas_id: 08
+    thomas_id: '08'
     address: H2-176 FHOB; Washington, DC 20515-6480
     phone: (202) 226-8417
   - name: Counterterrorism and Intelligence
@@ -298,7 +298,7 @@
     address: H2-176 FHOB; Washington, DC 20515-6480
     phone: (202) 226-8417
   - name: Oversight, Investigations, and Management
-    thomas_id: 09
+    thomas_id: '09'
     address: H2-176 FHOB; Washington, DC 20515-6480
     phone: (202) 226-8417
   - name: Transportation Security
@@ -373,7 +373,7 @@
   house_committee_id: IG
   subcommittees:
   - name: Oversight and Investigations
-    thomas_id: 09
+    thomas_id: '09'
     address: HVC304 CAPITOL; Washington, DC 20515
     phone: (202) 225-4121
   - name: Technical and Tactical Intelligence
@@ -381,7 +381,7 @@
     address: HVC304 CAPITOL; Washington, DC 20515
     phone: (202) 225-4121
   - name: Terrorism, Human Intelligence, Analysis, and Counterintelligence
-    thomas_id: 08
+    thomas_id: '08'
     address: HVC304 CAPITOL; Washington, DC 20515
     phone: (202) 225-4121
   address: HVC304 CAPITOL; Washington, DC 20515-6415
@@ -401,7 +401,7 @@
     address: 517 CHOB; Washington, DC 20515-6219
     phone: (202) 226-7680
   - name: Crime, Terrorism, and Homeland Security
-    thomas_id: 08
+    thomas_id: '08'
     address: B370B RHOB; Washington, DC 20515-6223
     phone: (202) 225-5727
   - name: Immigration Policy and Enforcement
@@ -532,7 +532,7 @@
   house_committee_id: VR
   subcommittees:
   - name: Disability Assistance and Memorial Affairs
-    thomas_id: 09
+    thomas_id: '09'
     address: 337 CHOB; Washington, DC 20515
     phone: (202) 225-9164
   - name: Economic Opportunity
@@ -544,7 +544,7 @@
     address: 338 CHOB; Washington, DC 20515-6340
     phone: (202) 225-9154
   - name: Oversight and Investigations
-    thomas_id: 08
+    thomas_id: '08'
     address: 337A CHOB; Washington, DC 20515-6339
     phone: (202) 225-3569
   address: 335 CHOB; Washington, DC 20515-6335
@@ -676,7 +676,7 @@
   - name: Department of Defense
     thomas_id: '02'
   - name: Legislative Branch
-    thomas_id: 08
+    thomas_id: '08'
 - type: senate
   name: Senate Committee on Armed Services
   url: http://armed-services.senate.gov/
@@ -708,9 +708,9 @@
   - name: Securities, Insurance and Investment
     thomas_id: '04'
   - name: Housing, Transportation and Community Development
-    thomas_id: 09
+    thomas_id: '09'
   - name: Financial Institutions and Consumer Protection
-    thomas_id: 08
+    thomas_id: '08'
 - type: senate
   name: Senate Committee on the Budget
   url: http://budget.senate.gov/
@@ -759,14 +759,14 @@
   - name: Clean Air and Nuclear Safety
     thomas_id: '10'
   - name: Superfund, Toxics and Environmental Health
-    thomas_id: 09
+    thomas_id: '09'
   - thomas_id: '17'
     name: Children's Health and Environmental Responsibility
   - thomas_id: '16'
     name: Green Jobs and the New Economy
   - thomas_id: '18'
     name: Oversight
-  - thomas_id: 08
+  - thomas_id: '08'
     name: Transportation and Infrastructure
   - thomas_id: '15'
     name: Water and Wildlife
@@ -803,7 +803,7 @@
   - name: European Affairs
     thomas_id: '01'
   - name: African Affairs
-    thomas_id: 09
+    thomas_id: '09'
   - name: East Asian and Pacific Affairs
     thomas_id: '02'
   - name: Near Eastern and South and Central Asian Affairs
@@ -823,7 +823,7 @@
     thomas_id: '03'
   - name: Federal Financial Management, Government Information, Federal Services,
       and International Security
-    thomas_id: 09
+    thomas_id: '09'
   - thomas_id: '13'
     name: Ad Hoc Subcommittee on Contracting Oversight
   - thomas_id: '01'
@@ -839,7 +839,7 @@
   - name: Primary Health and Aging
     thomas_id: '12'
   - name: Children and Families
-    thomas_id: 09
+    thomas_id: '09'
 - type: senate
   name: Senate Committee on the Judiciary
   url: http://judiciary.senate.gov/

committees-historical.yaml

@@ -207,7 +207,7 @@
     - 100
     - 101
   - name: Oceanography, Great Lakes and the Outer Continental Shelf
-    thomas_id: 08
+    thomas_id: '08'
     names:
       102: Oceanography, Great Lakes and the Outer Continental Shelf
       103: Oceanography, Gulf of Mexico and the Outer Continental Shelf
@@ -358,13 +358,13 @@
     - 101
     - 102
   - name: Ad Hoc Task Force on Presidential Pay Recommendation
-    thomas_id: 08
+    thomas_id: '08'
     names:
       100: Ad Hoc Task Force on Presidential Pay Recommendation
     congresses:
     - 100
   - name: Census, Statistics and Postal Personnel
-    thomas_id: 09
+    thomas_id: '09'
     names:
       103: Census, Statistics and Postal Personnel
     congresses:
@@ -721,7 +721,7 @@
       101: Tobacco and Peanuts
       102: Peanuts and Tobacco
   - name: Wheat, Soybeans, and Feed Grains
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -1102,7 +1102,7 @@
       104: Labor, Health and Human Services, and Education
       105: Labor, Health and Human Services, and Education
   - name: Legislative
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -1122,7 +1122,7 @@
       104: Legislative
       105: Legislative
   - name: Military Construction
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 97
     - 98
@@ -1592,7 +1592,7 @@
       102: Financial Institutions Supervision, Regulation and Insurance
       103: Financial Institutions Supervision, Regulation and Deposit Insurance
   - name: General Oversight and Renegotiation
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 97
     - 98
@@ -1652,7 +1652,7 @@
       99: International Development Institutions and Finance
       100: International Development Institutions and Finance
   - name: International Trade, Investment and Monetary Policy
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -1936,7 +1936,7 @@
       111: Health, Employment, Labor, and Pensions
       112: Health, Employment, Labor, and Pensions
   - name: Postsecondary Education
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -2100,7 +2100,7 @@
     112: Foreign Affairs
   subcommittees:
   - name: Africa
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -2528,7 +2528,7 @@
       107: Energy Policy, Natural Resources and Regulatory Affairs
       108: Energy Policy, Natural Resources and Regulatory Affairs
   - name: Civil Service
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 104
     - 105
@@ -2798,7 +2798,7 @@
       98: Services
       99: Services
   - name: Elections
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 99
     - 100
@@ -2816,7 +2816,7 @@
       110: Elections
       112: Elections
   - name: Libraries and Memorials
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 100
     - 101
@@ -2970,13 +2970,13 @@
     names:
       99: General Oversight, Northwest Power, and Forest Management
   - name: National Parks and Recreation
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 99
     names:
       99: National Parks and Recreation
   - name: Insular and International Affairs
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 101
     - 102
@@ -3570,7 +3570,7 @@
       102: Economic and Commercial Law
       103: Economic and Commercial Law
   - name: Crime and Criminal Justice
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 102
     - 103
@@ -3596,7 +3596,7 @@
       111: Crime, Terrorism, and Homeland Security
       112: Crime, Terrorism, and Homeland Security
   - name: Commercial and Administrative Law
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 104
     - 105
@@ -3854,7 +3854,7 @@
       111: Coast Guard and Maritime Transportation
       112: Coast Guard and Maritime Transportation
   - name: Public Buildings and Economic Development
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 104
     - 105
@@ -3865,7 +3865,7 @@
       106: Economic Development, Public Buildings, Hazardous Materials and Pipeline
         Transportation
   - name: Railroads
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 104
     - 105
@@ -4165,7 +4165,7 @@
       100: Transportation, Aviation and Materials
       101: Transportation, Aviation and Materials
   - name: Task Force on Science Policy
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 100
     names:
@@ -4467,7 +4467,7 @@
     names:
       104: Education, Training, Employment and Housing
   - name: Disability Assistance and Memorial Affairs
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 109
     - 110
@@ -4491,7 +4491,7 @@
       111: Economic Opportunity
       112: Economic Opportunity
   - name: Oversight and Investigations
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 109
     - 110
@@ -4870,7 +4870,7 @@
       102: Nutrition and Investigations
       103: Nutrition and Investigations
   - name: Forestry, Water Resources, and Environment
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     names:
@@ -5060,7 +5060,7 @@
       107: Interior
       108: Interior
   - name: Military Construction
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 97
     - 98
@@ -5276,7 +5276,7 @@
       107: Labor, Health and Human Services, and Education
       108: Labor, Health and Human Services, and Education
   - name: Legislative Branch
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 98
     - 99
@@ -5480,7 +5480,7 @@
     112: Armed Services
   subcommittees:
   - name: Manpower and Personnel
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 97
     - 98
@@ -5630,7 +5630,7 @@
     names:
       103: Defense Technology, Acquisition, and Industrial Base
   - name: Military Readiness and Defense Infrastructure
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 103
     names:
@@ -5828,7 +5828,7 @@
       98: Insurance
       105: Financial Services and Technology
   - name: Rural Housing and Development
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 98
     - 105
@@ -5848,7 +5848,7 @@
       111: Housing, Transportation and Community Development
       112: Housing, Transportation and Community Development
   - name: Financial Institutions and Regulatory Relief
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 104
     - 105
@@ -6053,7 +6053,7 @@
       102: Merchant Marine
       103: Merchant Marine
   - name: Surface Transportation
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -6087,7 +6087,7 @@
       98: National Ocean Policy Study
       99: National Ocean Policy Study
   - name: Science, Technology, and Space
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 98
     - 99
@@ -6397,7 +6397,7 @@
       104: Taxation and IRS Oversight
       105: Taxation and IRS Oversight
   - name: Medicare and Long-Term Care
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 101
     - 102
@@ -6413,7 +6413,7 @@
       105: Health Care
       106: Health Care
   - name: Long-Term Growth and Debt Reduction
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 106
     names:
@@ -6477,7 +6477,7 @@
       99: European Affairs
       108: European Affairs
   - name: African Affairs
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 98
     - 99
@@ -6683,7 +6683,7 @@
       100: Federal Spending, Budget, and Accounting
       102: Government Information and Regulation
   - name: Oversight of Government Management
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -6739,7 +6739,7 @@
       112: Oversight of Government Management, the Federal Workforce, and the District
         of Columbia
   - name: Financial Management, the Budget and International Security
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 108
     - 109
@@ -6929,7 +6929,7 @@
       111: Energy
       112: Energy
   - name: Public Lands, Reserved Water and Resource Conservation
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 97
     - 98
@@ -6939,7 +6939,7 @@
       98: Public Lands, Reserved Water and Resource Conservation
       99: Public Lands, Reserved Water and Resource Conservation
   - name: Water and Power
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -7269,7 +7269,7 @@
       104: Youth Violence
       105: Youth Violence
   - name: Technology, Terrorism, and Government Information
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 105
     - 106
@@ -7385,7 +7385,7 @@
       105: Public Health and Safety
       107: Public Health
   - name: Education, Arts, and Humanities
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 97
     - 98
@@ -7499,7 +7499,7 @@
       102: Labor
       103: Labor
   - name: Education and Early Childhood Development
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 109
     names:
@@ -7685,7 +7685,7 @@
       107: Fisheries, Wildlife, and Water
       108: Fisheries, Wildlife, and Water
   - name: Superfund, Waste Control, and Risk Assessment
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 104
     - 105
@@ -7699,7 +7699,7 @@
       107: Superfund, Toxics, Risk, and Waste Management
       109: Superfund and Waste Management
   - name: Transportation and Infrastructure
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 104
     - 105
@@ -7968,7 +7968,7 @@
       98: Antitrust and Restraint of Trade Activities Affecting Small Business
       99: Antitrust and Restraint of Trade Activities Affecting Small Business
   - name: Procurement, Tourism, and Rural Development
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 101
     - 102
@@ -8317,7 +8317,7 @@
       111: Transportation Security and Infrastructure Protection
       112: Transportation Security
   - name: Emergency Preparedness, Science, and Technology
-    thomas_id: 08
+    thomas_id: '08'
     congresses:
     - 109
     - 110
@@ -8329,7 +8329,7 @@
       111: Emerging Threats, Cybersecurity, and Science and Technology
       112: Cybersecurity, Infrastructure Protection, and Security Technologies
   - name: Management, Integration, and Oversight
-    thomas_id: 09
+    thomas_id: '09'
     congresses:
     - 109
     - 110

executive.yaml

@@ -574,7 +574,7 @@
     party: Republican
 - id:
     bioguide: F000260
-    thomas: 00399
+    thomas: '00399'
     govtrack: 404212
     icpsr: 3268
   name:
@@ -693,4 +693,3 @@
     start: '2013-01-20'
     end: '2017-01-20'
     party: Democrat
-

scripts/lint.py

@@ -0,0 +1,17 @@
+# Just loads and saves each .yaml file to normalize serialization syntax.
+#
+# python lint.py
+# ... will lint every .yaml file in the data directory.
+#
+# python lint.py file1.yaml file2.yaml ...
+# ... will lint the specified files.
+
+import glob, sys
+
+from utils import yaml_load, yaml_dump, data_dir
+
+for fn in glob.glob(data_dir() + "/*.yaml") if len(sys.argv) == 1 else sys.argv[1:]:
+	print fn + "..."
+	data = yaml_load(fn, use_cache=False)
+	yaml_dump(data, fn)
+	

scripts/utils.py

@@ -145,8 +145,6 @@ def unescape(text):
 # In order to preserve the order of attributes, YAML must be
 # hooked to load mappings as OrderedDicts. Adapted from:
 # https://gist.github.com/317164
-# Additionally, we need to set default output parameters
-# controlling formatting.
 
 import yaml
 from collections import OrderedDict
@@ -166,8 +164,39 @@ def construct_odict(load, node):
         omap[key] = value
 
 yaml.add_constructor(u'tag:yaml.org,2002:map', construct_odict)
+def ordered_dict_serializer(self, data):
+    return self.represent_mapping('tag:yaml.org,2002:map', data.items())
+yaml.add_representer(OrderedDict, ordered_dict_serializer)
 
-def yaml_load(path):
+# Likewise, when we store unicode objects make sure we don't write
+# them with weird YAML tags indicating the Python data type. The
+# standard string type is fine. We should do this:
+#   yaml.add_representer(unicode, lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:str', value))
+#
+# However, the standard PyYAML representer for strings does something
+# weird: if a value cannot be parsed as an integer quotes are omitted.
+#
+# This is incredibly odd when the value is an integer with a leading
+# zero. These values are typically parsed as octal integers, meaning
+# quotes would normally be required (that's good). But when the value
+# has an '8' or '9' in it, this would make it an invalid octal number
+# and so quotes would no longer be required (that's confusing).
+# We will override str and unicode output to choose the quotation
+# style with our own logic. (According to PyYAML, style can be one of
+# the empty string, ', ", |, or >, or None to, presumably, choose
+# automatically.
+def our_string_representer(dumper, value):
+	# If it looks like an octal number, force '-quote style.
+	style = None
+	if re.match(r"^0\d*$", value): style = "'"
+	return dumper.represent_scalar(u'tag:yaml.org,2002:str', value, style=style)
+yaml.add_representer(str, our_string_representer)
+yaml.add_representer(unicode, our_string_representer)
+        
+# Apply some common settings for loading/dumping YAML and cache the
+# data in pickled format which is a LOT faster than YAML.
+
+def yaml_load(path, use_cache=True):
     # Loading YAML is ridiculously slow, so cache the YAML data
     # in a pickled file which loads much faster.
 
@@ -175,12 +204,12 @@ def yaml_load(path):
     # matches the hash of the YAML file, and if so unpickle it.
     import cPickle as pickle, os.path, hashlib
     h = hashlib.sha1(open(path).read()).hexdigest()
-    if os.path.exists(path + ".pickle"):
+    if use_cache and os.path.exists(path + ".pickle"):
         store = pickle.load(open(path + ".pickle"))
         if store["hash"] == h:
             return store["data"]
-	
-	# No cached pickled data exists, so load the YAML file.
+    
+    # No cached pickled data exists, so load the YAML file.
     data = yaml.load(open(path))
     
     # Store in a pickled file for fast access later.
@@ -188,11 +217,6 @@ def yaml_load(path):
     
     return data
 
-def ordered_dict_serializer(self, data):
-    return self.represent_mapping('tag:yaml.org,2002:map', data.items())
-yaml.add_representer(OrderedDict, ordered_dict_serializer)
-yaml.add_representer(unicode, lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:str', value))
-
 def yaml_dump(data, path):
     yaml.dump(data, open(path, "w"), default_flow_style=False, allow_unicode=True)