From c10772e3f33ec2d14f7eb442f8af8661e6a3b876 Mon Sep 17 00:00:00 2001 From: Akash Patel <17132214+acxz@users.noreply.github.com> Date: Sun, 27 Feb 2022 20:13:50 -0500 Subject: [PATCH] make congress into a python package (#267) change directory structure to make python package conventional add setup.py file to specify deps guide users to use the installed `usc-run` command associated changes to other scripts make scripts installable when package is installed add a symlink for congress/run.py to run for backwards compat remove redundant requirements file --- .travis.yml | 4 +- Dockerfile | 4 +- README.md | 11 +-- {contrib => congress}/__init__.py | 0 congress/contrib/__init__.py | 0 {contrib => congress/contrib}/beanstalkd.py | 6 +- congress/run.py | 78 +++++++++++++++++++ congress/tasks/__init__.py | 0 .../tasks}/adler_wilkerson_bills.py | 2 +- {tasks => congress/tasks}/amendment_info.py | 4 +- {tasks => congress/tasks}/bill_info.py | 2 +- {tasks => congress/tasks}/bills.py | 5 +- .../tasks}/committee_meetings.py | 2 +- {tasks => congress/tasks}/govinfo.py | 6 +- {tasks => congress/tasks}/nomination_info.py | 2 +- {tasks => congress/tasks}/nominations.py | 4 +- {tasks => congress/tasks}/statutes.py | 20 +++-- .../tasks}/upcoming_house_floor.py | 4 +- {tasks => congress/tasks}/utils.py | 0 {tasks => congress/tasks}/vote_info.py | 2 +- {tasks => congress/tasks}/votes.py | 4 +- {tasks => congress/tasks}/voteview.py | 4 +- .../tasks}/voteview_codedoptions.csv | 0 requirements.txt | 12 --- run | 75 +----------------- scripts/bills.sh | 4 +- scripts/statutes.sh | 6 +- scripts/votes.sh | 52 ++++++------- scripts/voteview.sh | 4 +- setup.py | 45 +++++++++++ test/run | 2 +- 31 files changed, 198 insertions(+), 166 deletions(-) rename {contrib => congress}/__init__.py (100%) create mode 100644 congress/contrib/__init__.py rename {contrib => congress/contrib}/beanstalkd.py (98%) create mode 100755 congress/run.py create mode 100644 congress/tasks/__init__.py rename {tasks => congress/tasks}/adler_wilkerson_bills.py (99%) rename {tasks => congress/tasks}/amendment_info.py (98%) rename {tasks => congress/tasks}/bill_info.py (99%) rename {tasks => congress/tasks}/bills.py (99%) rename {tasks => congress/tasks}/committee_meetings.py (99%) rename {tasks => congress/tasks}/govinfo.py (99%) rename {tasks => congress/tasks}/nomination_info.py (99%) rename {tasks => congress/tasks}/nominations.py (97%) rename {tasks => congress/tasks}/statutes.py (96%) rename {tasks => congress/tasks}/upcoming_house_floor.py (99%) rename {tasks => congress/tasks}/utils.py (100%) rename {tasks => congress/tasks}/vote_info.py (99%) rename {tasks => congress/tasks}/votes.py (98%) rename {tasks => congress/tasks}/voteview.py (99%) rename {tasks => congress/tasks}/voteview_codedoptions.csv (100%) delete mode 100644 requirements.txt mode change 100755 => 120000 run create mode 100644 setup.py diff --git a/.travis.yml b/.travis.yml index 0442948..30a7a0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,12 @@ python: os: - linux -install: pip install -r requirements.txt +install: pip install . script: python test/run after_success: - pip install pyflakes - - pyflakes tasks/*.py | tee >(wc -l) + - pyflakes congress/tasks/*.py | tee >(wc -l) - pyflakes test/*.py | tee >(wc -l) notifications: diff --git a/Dockerfile b/Dockerfile index 05ccfd7..31e8d7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN mkdir -p /opt/theunitedstates.io/ ADD . /opt/theunitedstates.io/congress/ WORKDIR /opt/theunitedstates.io/congress/ -RUN pip install -r requirements.txt +RUN pip install . RUN echo "/opt/theunitedstates.io/congress/" > /usr/lib/python3.6/dist-packages/congress.pth @@ -48,4 +48,4 @@ RUN mkdir -p /congress WORKDIR /congress CMD [] -ENTRYPOINT ["/opt/theunitedstates.io/congress/run"] +ENTRYPOINT ["/opt/theunitedstates.io/congress/congress/run.py"] diff --git a/README.md b/README.md index 809c4e2..9a64820 100644 --- a/README.md +++ b/README.md @@ -43,17 +43,18 @@ It's recommended you use a `virtualenv` (virtual environment) for development. C python3 -m venv congress source congress/bin/activate ``` -Finally, with your virtual environment activated, install Python packages: +Finally, with your virtual environment activated, install the package, which +will automatically pull in the Python dependencies: ```bash -pip3 install -r requirements.txt +pip install . ``` ### Collecting the data The general form to start the scraping process is: - ./run [--force] [other options] + usc-run [--force] [other options] where data-type is one of: @@ -67,8 +68,8 @@ where data-type is one of: To get data for bills, resolutions, and amendments, run: ```bash -./run govinfo --bulkdata=BILLSTATUS -./run bills +usc-run govinfo --bulkdata=BILLSTATUS +usc-run bills ``` The bills script will output bulk data into a top-level `data` directory, then organized by Congress number, bill type, and bill number. Two data output files will be generated for each bill: a JSON version (data.json) and an XML version (data.xml). diff --git a/contrib/__init__.py b/congress/__init__.py similarity index 100% rename from contrib/__init__.py rename to congress/__init__.py diff --git a/congress/contrib/__init__.py b/congress/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/contrib/beanstalkd.py b/congress/contrib/beanstalkd.py similarity index 98% rename from contrib/beanstalkd.py rename to congress/contrib/beanstalkd.py index f31177e..a018939 100644 --- a/contrib/beanstalkd.py +++ b/congress/contrib/beanstalkd.py @@ -4,7 +4,7 @@ A module that monkey-patches the output_bill method to push the bill identifier onto a task queue after the data file has been written to disk. To use this module, invoke the bills scraper with the --patch option like so: - ./run bills --patch=contrib.beanstalkd + usc-run bills --patch=contrib.beanstalkd You must include a 'beakstalk' section in config.yml with this structure (though the values are up to you): @@ -34,9 +34,7 @@ import beanstalkc # The patch module is loaded after the task module is loaded, so all task # modules are on the import path. -import bills -import amendment_info -import vote_info +from congress.tasks import bills, amendment_info, vote_info __all__ = [ diff --git a/congress/run.py b/congress/run.py new file mode 100755 index 0000000..0db323f --- /dev/null +++ b/congress/run.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +import sys +import os +import traceback +import pprint as pp +import logging +import importlib + +# set global HTTP timeouts to 10 seconds +import socket + +def main(): + socket.setdefaulttimeout(10) + + CONGRESS_ROOT = os.path.dirname(os.path.abspath(__file__)) + + # name of the task comes first + task_name = sys.argv[1] + + # parse any command line flags off + options = {} + for arg in sys.argv[2:]: + if arg.startswith("--"): + + if "=" in arg: + key, value = arg.split('=') + else: + key, value = arg, True + + key = key.split("--")[1] + if value == 'True': + value = True + elif value == 'False': + value = False + options[key.lower()] = value + + + # configure logging + if options.get('debug', False): + log_level = "debug" + else: + log_level = options.get("log", "warn") + + if log_level not in ["debug", "info", "warn", "error"]: + print("Invalid log level (specify: debug, info, warn, error).") + sys.exit(1) + + if options.get('timestamps', False): + logging.basicConfig(format='%(asctime)s %(message)s', level=log_level.upper()) + else: + logging.basicConfig(format='%(message)s', level=log_level.upper()) + + + sys.path.append(os.path.join(CONGRESS_ROOT, "tasks")) + import utils + + try: + task_mod = __import__(task_name) + + if 'patch' in options: + patch_mod = importlib.import_module(options['patch']) + patch_func = getattr(patch_mod, 'patch', None) + if patch_func is None: + logging.error("You specified a --patch argument but the {} module does not contain a 'patch' function.".format(options['patch'])) + sys.exit(1) + elif not callable(patch_func): + logging.error("You specified a --patch argument but {}.patch is not callable".format(options['patch'])) + sys.exit(1) + else: + patch_mod.patch(task_name) + + task_mod.run(options) + except Exception as exception: + utils.admin(exception) + +if __name__ == "__main__": + main() diff --git a/congress/tasks/__init__.py b/congress/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tasks/adler_wilkerson_bills.py b/congress/tasks/adler_wilkerson_bills.py similarity index 99% rename from tasks/adler_wilkerson_bills.py rename to congress/tasks/adler_wilkerson_bills.py index 9b44e27..cac9607 100644 --- a/tasks/adler_wilkerson_bills.py +++ b/congress/tasks/adler_wilkerson_bills.py @@ -6,7 +6,7 @@ import csv import zipfile import datetime -import utils +from congress.tasks import utils def run(options): diff --git a/tasks/amendment_info.py b/congress/tasks/amendment_info.py similarity index 98% rename from tasks/amendment_info.py rename to congress/tasks/amendment_info.py index e8e4de7..4c4f797 100644 --- a/tasks/amendment_info.py +++ b/congress/tasks/amendment_info.py @@ -5,9 +5,9 @@ import time import json from lxml import etree -import utils +from congress.tasks import utils -from bill_info import sponsor_for as sponsor_for_bill, action_for +from congress.tasks.bill_info import sponsor_for as sponsor_for_bill, action_for def process_amendment(amdt_data, bill_id, options): amdt = build_amendment_json_dict(amdt_data, options) diff --git a/tasks/bill_info.py b/congress/tasks/bill_info.py similarity index 99% rename from tasks/bill_info.py rename to congress/tasks/bill_info.py index 2ff435e..e7cf7be 100644 --- a/tasks/bill_info.py +++ b/congress/tasks/bill_info.py @@ -1,4 +1,4 @@ -import utils +from congress.tasks import utils import logging import re import json diff --git a/tasks/bills.py b/congress/tasks/bills.py similarity index 99% rename from tasks/bills.py rename to congress/tasks/bills.py index bf4dbe4..837b68e 100644 --- a/tasks/bills.py +++ b/congress/tasks/bills.py @@ -4,10 +4,7 @@ import os import re import xmltodict -import bill_info -import amendment_info -import govinfo -import utils +from congress.tasks import bill_info, amendment_info, govinfo, utils def run(options): diff --git a/tasks/committee_meetings.py b/congress/tasks/committee_meetings.py similarity index 99% rename from tasks/committee_meetings.py rename to congress/tasks/committee_meetings.py index b753251..56492fa 100644 --- a/tasks/committee_meetings.py +++ b/congress/tasks/committee_meetings.py @@ -1,4 +1,4 @@ -import utils +from congress.tasks import utils import os.path import os import re diff --git a/tasks/govinfo.py b/congress/tasks/govinfo.py similarity index 99% rename from tasks/govinfo.py rename to congress/tasks/govinfo.py index 9428a97..6bd427f 100644 --- a/tasks/govinfo.py +++ b/congress/tasks/govinfo.py @@ -3,12 +3,12 @@ # https://www.govinfo.gov/sitemaps for a list of collections. # This service was formerly called "Fdsys." # -# ./run govinfo--collections=BILLS,STATUTE,... +# usc-run govinfo --collections=BILLS,STATUTE,... # Download bill text (from the BILLS collection; there's also a bulk # data BILLS collection but it has less in it), the Statues at Large, # and other documents from GovInfo.gov's non-bulk-data collections. # -# ./run govinfo --bulkdata=BILLSTATUS,FR,... +# usc-run govinfo --bulkdata=BILLSTATUS,FR,... # Download bill status, the Federal Register, and other documents # from GovInfo.gov's bulk data collections. (The BILLS collection occurs # both as a regular collection (bill text in multiple formats) and as @@ -46,7 +46,7 @@ import logging import os import os.path import zipfile -import utils +from congress.tasks import utils import rtyaml diff --git a/tasks/nomination_info.py b/congress/tasks/nomination_info.py similarity index 99% rename from tasks/nomination_info.py rename to congress/tasks/nomination_info.py index 7604ca7..42519c0 100644 --- a/tasks/nomination_info.py +++ b/congress/tasks/nomination_info.py @@ -1,4 +1,4 @@ -import utils +from congress.tasks import utils import logging import re import json diff --git a/tasks/nominations.py b/congress/tasks/nominations.py similarity index 97% rename from tasks/nominations.py rename to congress/tasks/nominations.py index 33fef49..77d5743 100644 --- a/tasks/nominations.py +++ b/congress/tasks/nominations.py @@ -1,11 +1,11 @@ -import utils +from congress.tasks import utils import os import os.path import re from lxml import html, etree import logging -import nomination_info +from congress.tasks import nomination_info def run(options): diff --git a/tasks/statutes.py b/congress/tasks/statutes.py similarity index 96% rename from tasks/statutes.py rename to congress/tasks/statutes.py index a41fa9e..1f7b652 100644 --- a/tasks/statutes.py +++ b/congress/tasks/statutes.py @@ -12,15 +12,15 @@ # # First download the Statutes at Large from GPO: # -# ./run fdsys --collections=STATUTE --store=mods +# usc-run fdsys --collections=STATUTE --store=mods # # To process statute text, get the text PDFs: # -# ./run fdsys --collections=STATUTE --store=pdfs --granules +# usc-run fdsys --collections=STATUTE --store=pdfs --granules # # Then run this script: # -# ./run statutes +# usc-run statutes # # Processes all downloaded statutes files and saves bill files: # data/82/bills/hr/hr1/data.json and @@ -41,10 +41,10 @@ # UTF-8 encoded and have form-feed characters marking page breaks. # # Examples: -# ./run statutes --volume=65 -# ./run statutes --volumes=65-86 -# ./run statutes --year=1951 -# ./run statutes --years=1951-1972 +# usc-run statutes --volume=65 +# usc-run statutes --volumes=65-86 +# usc-run statutes --year=1951 +# usc-run statutes --years=1951-1972 # Processes just the indicated volume or range of volumes. # Starting with the 93rd Congress (1973-1974, corresponding # to volume 78 of the Statutes of Large), we have bill @@ -52,7 +52,7 @@ # # With bill text missing from THOMAS/GPO from the 93rd to # 102nd Congresses, fill in the text-versions files like so: -# ./run statutes --volumes=87-106 --textversions +# usc-run statutes --volumes=87-106 --textversions import logging import time @@ -63,9 +63,7 @@ import json import os.path import subprocess -import utils -import bill_info -import bill_versions +from congress.tasks import utils, bill_info, bill_versions import fdsys diff --git a/tasks/upcoming_house_floor.py b/congress/tasks/upcoming_house_floor.py similarity index 99% rename from tasks/upcoming_house_floor.py rename to congress/tasks/upcoming_house_floor.py index e00310d..18e73ab 100644 --- a/tasks/upcoming_house_floor.py +++ b/congress/tasks/upcoming_house_floor.py @@ -1,4 +1,4 @@ -import utils +from congress.tasks import utils import logging import sys import os @@ -13,7 +13,7 @@ import subprocess from bs4 import BeautifulSoup -from bills import output_for_bill +from congress.tasks.bills import output_for_bill # Parsing data from the House' upcoming floor feed, at # https://docs.house.gov/floor/ diff --git a/tasks/utils.py b/congress/tasks/utils.py similarity index 100% rename from tasks/utils.py rename to congress/tasks/utils.py diff --git a/tasks/vote_info.py b/congress/tasks/vote_info.py similarity index 99% rename from tasks/vote_info.py rename to congress/tasks/vote_info.py index 17a8513..31b3719 100644 --- a/tasks/vote_info.py +++ b/congress/tasks/vote_info.py @@ -1,4 +1,4 @@ -import utils +from congress.tasks import utils import logging import re import json diff --git a/tasks/votes.py b/congress/tasks/votes.py similarity index 98% rename from tasks/votes.py rename to congress/tasks/votes.py index 02cd9cf..5c9383f 100644 --- a/tasks/votes.py +++ b/congress/tasks/votes.py @@ -1,4 +1,4 @@ -import utils +from congress.tasks import utils import json from iso8601 import iso8601 import datetime @@ -11,7 +11,7 @@ import datetime from lxml import html, etree import logging -import vote_info +from congress.tasks import vote_info def run(options): diff --git a/tasks/voteview.py b/congress/tasks/voteview.py similarity index 99% rename from tasks/voteview.py rename to congress/tasks/voteview.py index 043e9d3..5acb024 100644 --- a/tasks/voteview.py +++ b/congress/tasks/voteview.py @@ -5,8 +5,8 @@ import datetime import time import logging -import utils -from vote_info import output_vote +from congress.tasks import utils +from congress.tasks.vote_info import output_vote # load some hard-coded codes special_vote_options = { } diff --git a/tasks/voteview_codedoptions.csv b/congress/tasks/voteview_codedoptions.csv similarity index 100% rename from tasks/voteview_codedoptions.csv rename to congress/tasks/voteview_codedoptions.csv diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f7f65b9..0000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -pyyaml -iso8601 -python-dateutil -lxml>=2.2 -pytz -cssselect -scrapelib -mechanize -BeautifulSoup4 -mock -xmltodict -rtyaml diff --git a/run b/run deleted file mode 100755 index 3ea752f..0000000 --- a/run +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -import sys -import os -import traceback -import pprint as pp -import logging -import importlib - -# set global HTTP timeouts to 10 seconds -import socket -socket.setdefaulttimeout(10) - -CONGRESS_ROOT = os.path.dirname(os.path.abspath(__file__)) - -# name of the task comes first -task_name = sys.argv[1] - -# parse any command line flags off -options = {} -for arg in sys.argv[2:]: - if arg.startswith("--"): - - if "=" in arg: - key, value = arg.split('=') - else: - key, value = arg, True - - key = key.split("--")[1] - if value == 'True': - value = True - elif value == 'False': - value = False - options[key.lower()] = value - - -# configure logging -if options.get('debug', False): - log_level = "debug" -else: - log_level = options.get("log", "warn") - -if log_level not in ["debug", "info", "warn", "error"]: - print("Invalid log level (specify: debug, info, warn, error).") - sys.exit(1) - -if options.get('timestamps', False): - logging.basicConfig(format='%(asctime)s %(message)s', level=log_level.upper()) -else: - logging.basicConfig(format='%(message)s', level=log_level.upper()) - - -sys.path.append(os.path.join(CONGRESS_ROOT, "tasks")) -import utils - -try: - task_mod = __import__(task_name) - - if 'patch' in options: - patch_mod = importlib.import_module(options['patch']) - patch_func = getattr(patch_mod, 'patch', None) - if patch_func is None: - logging.error("You specified a --patch argument but the {} module does not contain a 'patch' function.".format(options['patch'])) - sys.exit(1) - elif not callable(patch_func): - logging.error("You specified a --patch argument but {}.patch is not callable".format(options['patch'])) - sys.exit(1) - else: - patch_mod.patch(task_name) - - task_mod.run(options) -except Exception as exception: - utils.admin(exception) - diff --git a/run b/run new file mode 120000 index 0000000..1ce9c92 --- /dev/null +++ b/run @@ -0,0 +1 @@ +congress/run.py \ No newline at end of file diff --git a/scripts/bills.sh b/scripts/bills.sh index 21ebac3..785bb27 100755 --- a/scripts/bills.sh +++ b/scripts/bills.sh @@ -1,6 +1,6 @@ #!/bin/sh # Refresh the bulk data collection. -./run govinfo --bulkdata=BILLSTATUS +usc-run govinfo --bulkdata=BILLSTATUS # Turn into JSON and GovTrack-XML. -./run bills --govtrack $@ +usc-run bills --govtrack $@ diff --git a/scripts/statutes.sh b/scripts/statutes.sh index da65a83..6108060 100755 --- a/scripts/statutes.sh +++ b/scripts/statutes.sh @@ -1,4 +1,4 @@ #!/bin/sh -./run govinfo --collections=STATUTE --extract=mods,pdf -./run statutes --volumes=65-86 --govtrack # bill status -./run statutes --volumes=65-106 --textversions --extracttext # bill text +usc-run govinfo --collections=STATUTE --extract=mods,pdf +usc-run statutes --volumes=65-86 --govtrack # bill status +usc-run statutes --volumes=65-106 --textversions --extracttext # bill text diff --git a/scripts/votes.sh b/scripts/votes.sh index f847e6c..7a8afcc 100755 --- a/scripts/votes.sh +++ b/scripts/votes.sh @@ -1,27 +1,27 @@ #!/bin/sh -./run votes --govtrack --congress=101 --session=1989 --chamber=senate $@ -./run votes --govtrack --congress=101 --session=1990 $@ -./run votes --govtrack --congress=102 --session=1991 $@ -./run votes --govtrack --congress=102 --session=1992 $@ -./run votes --govtrack --congress=103 --session=1993 $@ -./run votes --govtrack --congress=103 --session=1994 $@ -./run votes --govtrack --congress=104 --session=1995 $@ -./run votes --govtrack --congress=104 --session=1996 $@ -./run votes --govtrack --congress=105 --session=1997 $@ -./run votes --govtrack --congress=105 --session=1998 $@ -./run votes --govtrack --congress=106 --session=1999 $@ -./run votes --govtrack --congress=106 --session=2000 $@ -./run votes --govtrack --congress=107 --session=2001 $@ -./run votes --govtrack --congress=107 --session=2002 $@ -./run votes --govtrack --congress=108 --session=2003 $@ -./run votes --govtrack --congress=108 --session=2004 $@ -./run votes --govtrack --congress=109 --session=2005 $@ -./run votes --govtrack --congress=109 --session=2006 $@ -./run votes --govtrack --congress=110 --session=2007 $@ -./run votes --govtrack --congress=110 --session=2008 $@ -./run votes --govtrack --congress=111 --session=2009 $@ -./run votes --govtrack --congress=111 --session=2010 $@ -./run votes --govtrack --congress=112 --session=2011 $@ -./run votes --govtrack --congress=112 --session=2012 $@ -./run votes --govtrack --congress=113 --session=2013 $@ -./run votes --govtrack --congress=113 --session=2014 $@ +usc-run votes --govtrack --congress=101 --session=1989 --chamber=senate $@ +usc-run votes --govtrack --congress=101 --session=1990 $@ +usc-run votes --govtrack --congress=102 --session=1991 $@ +usc-run votes --govtrack --congress=102 --session=1992 $@ +usc-run votes --govtrack --congress=103 --session=1993 $@ +usc-run votes --govtrack --congress=103 --session=1994 $@ +usc-run votes --govtrack --congress=104 --session=1995 $@ +usc-run votes --govtrack --congress=104 --session=1996 $@ +usc-run votes --govtrack --congress=105 --session=1997 $@ +usc-run votes --govtrack --congress=105 --session=1998 $@ +usc-run votes --govtrack --congress=106 --session=1999 $@ +usc-run votes --govtrack --congress=106 --session=2000 $@ +usc-run votes --govtrack --congress=107 --session=2001 $@ +usc-run votes --govtrack --congress=107 --session=2002 $@ +usc-run votes --govtrack --congress=108 --session=2003 $@ +usc-run votes --govtrack --congress=108 --session=2004 $@ +usc-run votes --govtrack --congress=109 --session=2005 $@ +usc-run votes --govtrack --congress=109 --session=2006 $@ +usc-run votes --govtrack --congress=110 --session=2007 $@ +usc-run votes --govtrack --congress=110 --session=2008 $@ +usc-run votes --govtrack --congress=111 --session=2009 $@ +usc-run votes --govtrack --congress=111 --session=2010 $@ +usc-run votes --govtrack --congress=112 --session=2011 $@ +usc-run votes --govtrack --congress=112 --session=2012 $@ +usc-run votes --govtrack --congress=113 --session=2013 $@ +usc-run votes --govtrack --congress=113 --session=2014 $@ diff --git a/scripts/voteview.sh b/scripts/voteview.sh index 13d54d5..ea56655 100755 --- a/scripts/voteview.sh +++ b/scripts/voteview.sh @@ -1,8 +1,8 @@ for congress in {1..100}; do - ./run voteview --congress=$congress --govtrack $@ + usc-run voteview --congress=$congress --govtrack $@ # After the first run, no need to update legislator info. export UPDATE_CONGRESS_LEGISLATORS=NO done -./run voteview --govtrack --congress=101 --session=1989 --chamber=h $@ +usc-run voteview --govtrack --congress=101 --session=1989 --chamber=h $@ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..97b405f --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +"""Setup file for using congress as a python package.""" +from os import path + +import setuptools + +# Obtain long_description from README.md +here = path.abspath(path.dirname(__file__)) +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setuptools.setup( + name='united-states-congress', + version='0.0.1', + author='The unitedstates organization on GitHub', + long_description=long_description, + description='Public domain data collectors for the work of Congress, ' + 'including legislation, amendments, and votes.', + license='CC0-1.0', + packages=setuptools.find_packages(), + install_requires=[ + 'beautifulsoup4', + 'cssselect', + 'iso8601', + 'lxml', + 'mechanize', + 'mock', + 'rtyaml', + 'python-dateutil', + 'pytz', + 'pyyaml', + 'scrapelib', + 'xmltodict', + ], + entry_points={ + 'console_scripts': [ + 'usc-run=congress.run:main' + ], + }, + scripts=[ + 'scripts/bills.sh', + 'scripts/statutes.sh', + 'scripts/votes.sh', + 'scripts/voteview.sh' + ], +) diff --git a/test/run b/test/run index b08e294..ba2743a 100755 --- a/test/run +++ b/test/run @@ -2,7 +2,7 @@ import sys import unittest -sys.path.append("tasks") # allow test classes to easily load tasks +sys.path.append("congress/tasks") # allow test classes to easily load tasks sys.path.append("test") # allow fixtures.py to be loaded tests = unittest.TestLoader().discover("test")