[^/]+)/?$', path)
- if host in ('www.github.com', 'github.com') and matched:
- projects_url = GITHUB_USER_REPOS_API_URL.format(username=matched.group('name'))
+ github_username = parse_github_user(organization.projects_list_url)
+ if github_username:
+ projects_url = GITHUB_USER_REPOS_API_URL.format(username=github_username)
try:
- response = get_github_api(projects_url)
+ got = get_github_api(projects_url)
# Consider any status other than 2xx an error
- if not response.status_code // 100 == 2:
+ if not got.status_code // 100 == 2:
return []
- projects = get_adjoined_json_lists(response)
+ projects, _ = get_adjoined_json_lists(got)
except exceptions.RequestException:
# Something has gone wrong, probably a bad URL or site is down.
@@ -255,6 +409,10 @@ def get_projects(organization):
# some values might be empty strings
elif type(project_value) in (str, unicode) and unicode(project_value.decode('utf8')) == u'':
project[project_key] = None
+ # we want tags to be a list with no whitespace
+ elif project_key == 'tags':
+ project_value = unicode(project_value.decode('utf8'))
+ project[project_key] = [tag.strip() for tag in project_value.split(',')]
else:
project[project_key] = unicode(project_value.decode('utf8'))
@@ -299,13 +457,42 @@ def get_projects(organization):
return projects
+
+def github_latest_update_time(github_details):
+ '''
+ Use `pushed_at` date, if present, which is updated any time any
+ branch is pushed. This tends to be more similar to the first
+ date visible when users click through to the project -- the
+ last commit's date. If there is no pushed_at date then fall back to `updated_at` date.
+
+ It's still not perfect, but this will be a quick improvement to avoid the
+ confusion of seeing a "last modified yesterday" project that actually
+ hasn't seen a commit in three years.
+
+ (See issue #245 for some context, but we ripped it out)
+ '''
+ import dateutil.parser
+
+ datetime_format = '%a, %d %b %Y %H:%M:%S %Z'
+
+ if 'pushed_at' in github_details:
+ update_time = github_details['pushed_at']
+ elif 'updated_at' in github_details:
+ update_time = github_details['updated_at']
+ else:
+ return datetime.now()
+
+ return dateutil.parser.parse(update_time).strftime(datetime_format)
+
+
def non_github_project_update_time(project):
''' If its a non-github project, we should check if any of the fields
have been updated, such as the description.
Set the last_updated timestamp.
'''
- existing_project = db.session.query(Project).filter(Project.name == project['name']).first()
+ filters = [Project.name == project['name'], Project.organization_name == project['organization_name']]
+ existing_project = db.session.query(Project).filter(*filters).first()
if existing_project:
# project gets existing last_updated
@@ -322,6 +509,17 @@ def non_github_project_update_time(project):
return project
+
+def make_root_github_project_path(path):
+ ''' Strip anything extra off the end of a github path
+ '''
+ path_split = path.split('/')
+ path = '/'.join(path_split[0:3])
+ # some URLs have been passed to us with '.git' at the end
+ path = sub(ur'\.git$', '', path)
+ return path
+
+
def update_project_info(project):
''' Update info from Github, if it's missing.
@@ -345,21 +543,30 @@ def update_project_info(project):
# Get the Github attributes
if host == 'github.com':
+ path = sub(r"[\s\/]+?$", "", path)
+ # make sure we're working with the main github URL
+ path = make_root_github_project_path(path)
repo_url = GITHUB_REPOS_API_URL.format(repo_path=path)
- # If we've hit the GitHub rate limit, skip updating projects.
- global github_throttling
- if github_throttling:
- return project
-
# find an existing project, filtering on code_url, organization_name, and project name (if we know it)
existing_filter = [Project.code_url == project['code_url'], Project.organization_name == project['organization_name']]
if 'name' in project and project['name']:
existing_filter.append(Project.name == project['name'])
+ existing_project = db.session.query(Project).filter(*existing_filter).first()
+
+ # if we're throttled, make sure an existing project is kept and return none
+ if GITHUB_THROTTLING:
+ if existing_project:
+ # :::here (project/true)
+ existing_project.keep = True
+ # commit the project
+ db.session.commit()
+ return None
+
+ # keep track of org spreadsheet values
spreadsheet_is_updated = False
- existing_project = db.session.query(Project).filter(*existing_filter).first()
if existing_project:
# copy 'last_updated' values from the existing project to the project dict
project['last_updated'] = existing_project.last_updated
@@ -373,6 +580,7 @@ def update_project_info(project):
existing_value = existing_project.__dict__[project_key]
if check_value and check_value != existing_value:
spreadsheet_is_updated = True
+ project[project_key] = check_value
elif not check_value and existing_value:
project[project_key] = existing_value
@@ -390,55 +598,53 @@ def update_project_info(project):
if got.status_code in range(400, 499):
if got.status_code == 404:
- logging.error(repo_url + ' doesn\'t exist.')
- # If its a bad GitHub link, don't return it at all.
+ # It's a bad GitHub link
+ logging.error(u"{} doesn't exist.".format(repo_url))
+ # If there's an existing project in the database, get rid of it
+ if existing_project:
+ # this is redundant, but let's make sure
+ # :::here (project/false)
+ existing_project.keep = False
+ db.session.commit()
+ # Take the project out of the loop by returning None
return None
+
elif got.status_code == 403:
- logging.error("GitHub Rate Limit Remaining: " + str(got.headers["x-ratelimit-remaining"]))
- error_dict = {
- "error": u'IOError: We done got throttled by GitHub',
- "time": datetime.now()
- }
- new_error = Error(**error_dict)
- db.session.add(new_error)
- # commit the error
- db.session.commit()
- github_throttling = True
- return project
+ # Throttled by GitHub
+ if existing_project:
+ # :::here (project/true)
+ existing_project.keep = True
+ # commit the project
+ db.session.commit()
+ return None
else:
raise IOError
# If the project has not been modified...
elif got.status_code == 304:
- logging.info('Project {} has not been modified since last update'.format(repo_url))
-
- # Populate values from the civic.json if it exists/is updated
- project, civic_json_is_updated = update_project_from_civic_json(project_dict=project, force=spreadsheet_is_updated)
+ logging.info(u'Project {} has not been modified since last update'.format(repo_url))
# if values have changed, copy untouched values from the existing project object and return it
- if spreadsheet_is_updated or civic_json_is_updated:
- logging.info('Project %s has been modified via spreadsheet or civic.json.', repo_url)
- project['last_updated'] = datetime.now().strftime("%a, %d %b %Y %H:%M:%S %Z")
+ if spreadsheet_is_updated:
+ logging.info('Project %s has been modified via spreadsheet.', repo_url)
project['github_details'] = existing_project.github_details
return project
# nothing was updated, but make sure we keep the project
# :::here (project/true)
existing_project.keep = True
- db.session.add(existing_project)
# commit the project
db.session.commit()
return None
- # Save last_updated time header for future requests
- project['last_updated'] = got.headers['Last-Modified']
-
+ # the project has been modified
all_github_attributes = got.json()
github_details = {}
for field in ('contributors_url', 'created_at', 'forks_count', 'homepage',
- 'html_url', 'id', 'language', 'open_issues', 'pushed_at',
- 'updated_at', 'watchers_count', 'name', 'description', 'stargazers_count'):
+ 'html_url', 'id', 'open_issues', 'pushed_at',
+ 'updated_at', 'watchers_count', 'name', 'description',
+ 'stargazers_count', 'subscribers_count'):
github_details[field] = all_github_attributes[field]
github_details['owner'] = dict()
@@ -457,15 +663,24 @@ def update_project_info(project):
if 'link_url' not in project or not project['link_url']:
project['link_url'] = all_github_attributes['homepage']
+ project['last_updated'] = github_latest_update_time(github_details)
+
+ # Grab the list of project languages
+ got = get_github_api(all_github_attributes['languages_url'])
+ languages_json = got.json()
+ if got.status_code // 100 == 2 and languages_json.keys():
+ project['languages'] = languages_json.keys()
+ else:
+ project['languages'] = None
+
#
# Populate project contributors from github_details[contributors_url]
#
project['github_details']['contributors'] = []
got = get_github_api(all_github_attributes['contributors_url'])
-
- # Check if there are contributors
try:
- for contributor in got.json():
+ contributors_json = got.json()
+ for contributor in contributors_json:
# we don't want people without email addresses?
if contributor['login'] == 'invalid-email-address':
break
@@ -487,7 +702,8 @@ def update_project_info(project):
#
got = get_github_api(all_github_attributes['url'] + '/stats/participation')
try:
- project['github_details']['participation'] = got.json()['all']
+ participation_json = got.json()
+ project['github_details']['participation'] = participation_json['all']
except:
project['github_details']['participation'] = [0] * 50
@@ -496,8 +712,16 @@ def update_project_info(project):
#
project, civic_json_is_updated = update_project_from_civic_json(project_dict=project, force=spreadsheet_is_updated)
+ # Get the lastest commit status
+ # First build up the url to use
+ if "default_branch" in all_github_attributes:
+ commit_status_url = GITHUB_COMMIT_STATUS_URL.format(repo_path=path, default_branch=all_github_attributes['default_branch'])
+ got = get_github_api(commit_status_url)
+ project["commit_status"] = got.json().get('state', None)
+
return project
+
def extract_tag_value(tag_candidate):
''' Extract the value of a tag from a string or object. tag_candidate must
be in the form of either u'tag value' or {'tag': u'tag value'}
@@ -516,6 +740,7 @@ def extract_tag_value(tag_candidate):
return None
+
def get_tags_from_civic_json_object(tags_in):
''' Extract and return tags in the correct format from the passed object
'''
@@ -525,10 +750,9 @@ def get_tags_from_civic_json_object(tags_in):
# get the tags
extracted = [extract_tag_value(item) for item in tags_in]
- # strip None values
- stripped = [item for item in extracted if item is not None]
- # return as a string
- return u','.join(stripped) if len(stripped) else None
+ # strip None values and return as a list
+ return [item for item in extracted if item is not None]
+
def update_project_from_civic_json(project_dict, force=False):
''' Update and return the passed project dict with values from civic.json
@@ -554,6 +778,7 @@ def update_project_from_civic_json(project_dict, force=False):
return project_dict, is_updated
+
def get_issues_for_project(project):
''' get the issues for a single project in dict format
without touching the database (used for testing)
@@ -565,86 +790,102 @@ def get_issues_for_project(project):
# Get github issues api url
_, host, path, _, _, _ = urlparse(project.code_url)
+ path = sub(r"[\s\/]+?$", "", path)
+ # make sure we're working with the main github URL
+ path = make_root_github_project_path(path)
issues_url = GITHUB_ISSUES_API_URL.format(repo_path=path)
# Ping github's api for project issues
got = get_github_api(issues_url, headers={'If-None-Match': project.last_updated_issues})
+ if got.status_code // 100 != 2:
+ return issues
# Save each issue in response
- responses = get_adjoined_json_lists(got, headers={'If-None-Match': project.last_updated_issues})
+ responses, _ = get_adjoined_json_lists(got, headers={'If-None-Match': project.last_updated_issues})
for issue in responses:
# Type check the issue, we are expecting a dictionary
if isinstance(issue, dict):
# Pull requests are returned along with issues. Skip them.
if "/pull/" in issue['html_url']:
continue
- issue_dict = dict(title=issue['title'], html_url=issue['html_url'],
- body=issue['body'], project_id=project.id, labels=issue['labels'])
+
+ issue_dict = dict(project_id=project.id)
+ for field in (
+ 'title', 'html_url', 'body',
+ 'labels', 'created_at', 'updated_at'):
+ issue_dict[field] = issue.get(field, None)
+
issues.append(issue_dict)
else:
logging.error('Issue for project %s is not a dictionary', project.name)
return issues
-def get_issues(org_name):
- '''
- Get github issues associated to each Organization's Projects.
+
+def get_issues(project):
+ ''' Get github issues associated with the passed Project.
'''
issues = []
- # Only grab this organization's projects
- projects = db.session.query(Project).filter(Project.organization_name == org_name).all()
+ # don't try to parse an empty code_url
+ if not project.code_url:
+ return issues
- # Populate issues for each project
- for project in projects:
- # Mark this project's issues for deletion
- # :::here (issue/false)
- db.session.execute(db.update(Issue, values={'keep': False}).where(Issue.project_id == project.id))
+ # Mark this project's issues for deletion
+ # :::here (issue/false)
+ db.session.execute(db.update(Issue, values={'keep': False}).where(Issue.project_id == project.id))
- # don't try to parse an empty code_url
- if not project.code_url:
- continue
+ # Get github issues api url
+ _, host, path, _, _, _ = urlparse(project.code_url)
- # Get github issues api url
- _, host, path, _, _, _ = urlparse(project.code_url)
+ # Only check issues if its a github project
+ if host != 'github.com':
+ return issues
- # Only check issues if its a github project
- if host != 'github.com':
- continue
+ path = sub(r"[\s\/]+?$", "", path)
+ # make sure we're working with the main github URL
+ path = make_root_github_project_path(path)
+ issues_url = GITHUB_ISSUES_API_URL.format(repo_path=path)
+
+ # Ping github's api for project issues
+ # :TODO: non-github projects are hitting here and shouldn't be!
+ got = get_github_api(issues_url, headers={'If-None-Match': project.last_updated_issues})
+
+ # A 304 means that issues have not been modified since we last checked
+ if got.status_code == 304:
+ # :::here (issue/true)
+ db.session.execute(db.update(Issue, values={'keep': True}).where(Issue.project_id == project.id))
+ logging.info('Issues %s have not changed since last update', issues_url)
+
+ elif got.status_code not in range(400, 499):
+ # Update the project's last_updated_issue field
+ project.last_updated_issues = unicode(got.headers['ETag'])
+ db.session.add(project)
+
+ # Get all the pages of issues
+ responses, _ = get_adjoined_json_lists(got)
+
+ # Save each issue in response
+ for issue in responses:
+ # Type check the issue, we are expecting a dictionary
+ if isinstance(issue, dict):
+ # Pull requests are returned along with issues. Skip them.
+ if "/pull/" in issue['html_url']:
+ continue
+
+ issue_dict = dict(project_id=project.id)
+ for field in (
+ 'title', 'html_url', 'body',
+ 'labels', 'created_at', 'updated_at'):
+ issue_dict[field] = issue.get(field, None)
+
+ issues.append(issue_dict)
+ else:
+ logging.error('Issue for project %s is not a dictionary', project.name)
- issues_url = GITHUB_ISSUES_API_URL.format(repo_path=path)
-
- # Ping github's api for project issues
- # :TODO: non-github projects are hitting here and shouldn't be!
- got = get_github_api(issues_url, headers={'If-None-Match': project.last_updated_issues})
-
- # Verify that content has not been modified since last run
- if got.status_code == 304:
- # :::here (issue/true)
- db.session.execute(db.update(Issue, values={'keep': True}).where(Issue.project_id == project.id))
- logging.info('Issues %s have not changed since last update', issues_url)
-
- elif got.status_code not in range(400, 499):
- # Update project's last_updated_issue field
- project.last_updated_issues = unicode(got.headers['ETag'])
- db.session.add(project)
-
- responses = get_adjoined_json_lists(got, headers={'If-None-Match': project.last_updated_issues})
-
- # Save each issue in response
- for issue in responses:
- # Type check the issue, we are expecting a dictionary
- if isinstance(issue, dict):
- # Pull requests are returned along with issues. Skip them.
- if "/pull/" in issue['html_url']:
- continue
- issue_dict = dict(title=issue['title'], html_url=issue['html_url'],
- body=issue['body'], project_id=project.id, labels=issue['labels'])
- issues.append(issue_dict)
- else:
- logging.error('Issue for project %s is not a dictionary', project.name)
return issues
+
def get_root_directory_listing_for_project(project_dict, force=False):
''' Get a listing of the project's github repo root directory. Will return
an empty list if the listing hasn't changed since the last time we asked
@@ -657,6 +898,9 @@ def get_root_directory_listing_for_project(project_dict, force=False):
# Get the API URL
_, host, path, _, _, _ = urlparse(project_dict['code_url'])
+ path = sub(r"[\s\/]+?$", "", path)
+ # make sure we're working with the main github URL
+ path = make_root_github_project_path(path)
directory_url = GITHUB_CONTENT_API_URL.format(repo_path=path, file_path='')
# Request the directory listing
@@ -667,20 +911,21 @@ def get_root_directory_listing_for_project(project_dict, force=False):
# Verify that content has not been modified since last run
if got.status_code == 304:
- logging.info('root directory listing has not changed since last update for {}'.format(directory_url))
+ logging.info(u'root directory listing has not changed since last update for {}'.format(directory_url))
elif got.status_code not in range(400, 499):
- logging.info('root directory listing has changed for {}'.format(directory_url))
+ logging.info(u'root directory listing has changed for {}'.format(directory_url))
# Update the project's last_updated_root_files field
project_dict['last_updated_root_files'] = unicode(got.headers['ETag'])
# get the contents of the file
listing = got.json()
else:
- logging.info('NO root directory listing found for {}'.format(directory_url))
+ logging.info(u'NO root directory listing found for {}'.format(directory_url))
return listing
+
def get_civic_json_exists_for_project(project_dict, force=False):
''' Return True if the passed project has a civic.json file in its root directory.
'''
@@ -688,6 +933,7 @@ def get_civic_json_exists_for_project(project_dict, force=False):
exists = 'civic.json' in [item['name'] for item in directory_listing]
return exists
+
def get_civic_json_for_project(project_dict, force=False):
''' Get the contents of the civic.json at the project's github repo root, if it exists.
'''
@@ -699,6 +945,9 @@ def get_civic_json_for_project(project_dict, force=False):
# Get the API URL (if 'code_url' wasn't in project_dict, it would've been caught upstream)
_, host, path, _, _, _ = urlparse(project_dict['code_url'])
+ path = sub(r"[\s\/]+?$", "", path)
+ # make sure we're working with the main github URL
+ path = make_root_github_project_path(path)
civic_url = GITHUB_CONTENT_API_URL.format(repo_path=path, file_path='civic.json')
# Request the contents of the civic.json file
@@ -711,23 +960,24 @@ def get_civic_json_for_project(project_dict, force=False):
# Verify that content has not been modified since last run
if got.status_code == 304:
- logging.info('Unchanged civic.json at {}'.format(civic_url))
+ logging.info(u'Unchanged civic.json at {}'.format(civic_url))
elif got.status_code not in range(400, 499):
- logging.info('New civic.json at {}'.format(civic_url))
+ logging.info(u'New civic.json at {}'.format(civic_url))
# Update the project's last_updated_civic_json field
project_dict['last_updated_civic_json'] = unicode(got.headers['ETag'])
try:
# get the contents of the file
civic = got.json()
except ValueError:
- logging.error('Malformed civic.json at {}'.format(civic_url))
+ logging.error(u'Malformed civic.json at {}'.format(civic_url))
else:
- logging.info('No civic.json at {}'.format(civic_url))
+ logging.info(u'No civic.json at {}'.format(civic_url))
return civic
+
def count_people_totals(all_projects):
''' Create a list of people details based on project details.
@@ -773,19 +1023,23 @@ def count_people_totals(all_projects):
return users
-def save_organization_info(session, org_dict):
+
+def save_organization_info(session, org_info):
''' Save a dictionary of organization info to the datastore session.
Return an app.Organization instance.
'''
+ # Set any empty strings in org_info to None
+ org_info = {key: None if not value else value for (key, value) in org_info.iteritems()}
+
# Select an existing organization by name.
- filter = Organization.name == org_dict['name']
+ filter = Organization.name == org_info['name']
existing_org = session.query(Organization).filter(filter).first()
# :::here (organization/true)
# If this is a new organization, save and return it. The keep parameter is True by default.
if not existing_org:
- new_organization = Organization(**org_dict)
+ new_organization = Organization(**org_info)
session.add(new_organization)
return new_organization
@@ -799,17 +1053,18 @@ def save_organization_info(session, org_dict):
existing_org.keep = True
# Update existing organization details.
- for (field, value) in org_dict.items():
+ for (field, value) in org_info.items():
setattr(existing_org, field, value)
return existing_org
+
def save_project_info(session, proj_dict):
''' Save a dictionary of project info to the datastore session.
Return an app.Project instance.
'''
- # Select the current project, filtering on name AND organization.
+ # Select the current project, filtering on name and organization.
filter = Project.name == proj_dict['name'], Project.organization_name == proj_dict['organization_name']
existing_project = session.query(Project).filter(*filter).first()
@@ -829,47 +1084,54 @@ def save_project_info(session, proj_dict):
return existing_project
-def save_issue(session, issue):
- '''
- Save a dictionary of issue info to the datastore session.
+
+def save_issue_info(session, issue_dict):
+ ''' Save a dictionary of issue info to the datastore session.
+
Return an app.Issue instance
'''
- # Select the current issue, filtering on title AND project_id.
- filter = Issue.title == issue['title'], Issue.project_id == issue['project_id']
+ # Select the current issue, filtering on html_url and project id.
+ filter = Issue.html_url == issue_dict['html_url'], Issue.project_id == issue_dict['project_id']
existing_issue = session.query(Issue).filter(*filter).first()
- # If this is a new issue save it
+ # If this is a new issue save and return it.
if not existing_issue:
- new_issue = Issue(**issue)
+ new_issue = Issue(**issue_dict)
session.add(new_issue)
- else:
- # Preserve the existing issue.
- # :::here (issue/true)
- existing_issue.keep = True
- # Update existing issue details
- existing_issue.title = issue['title']
- existing_issue.body = issue['body']
- existing_issue.html_url = issue['html_url']
- existing_issue.project_id = issue['project_id']
-
-def save_labels(session, issue):
- '''
- Save labels to issues
+ return new_issue
+
+ # Preserve the existing issue.
+ # :::here (issue/true)
+ existing_issue.keep = True
+
+ # Update existing issue details, skipping 'labels'
+ for (field, value) in issue_dict.items():
+ if field != 'labels':
+ setattr(existing_issue, field, value)
+
+ return existing_issue
+
+
+def save_labels_info(session, issue_dict):
+ ''' Save labels to issues
'''
- # Select the current issue, filtering on title AND project_id.
- filter = Issue.title == issue['title'], Issue.project_id == issue['project_id']
+ # Select the current issue, filtering on html_url and project id.
+ filter = Issue.html_url == issue_dict['html_url'], Issue.project_id == issue_dict['project_id']
existing_issue = session.query(Issue).filter(*filter).first()
# Get list of existing and incoming label names (dupes will be filtered out in comparison process)
existing_label_names = [label.name for label in existing_issue.labels]
- incoming_label_names = [label['name'] for label in issue['labels']]
+ incoming_label_names = [label['name'] for label in issue_dict['labels']]
# Add labels that are in the incoming list and not the existing list
add_label_names = list(set(incoming_label_names) - set(existing_label_names))
- for label_dict in issue['labels']:
+ for label_dict in issue_dict['labels']:
if label_dict['name'] in add_label_names:
# add the issue id to the labels
label_dict["issue_id"] = existing_issue.id
+ # remove id and default from some labels
+ label_dict.pop("default", None)
+ label_dict.pop("id", None)
new_label = Label(**label_dict)
session.add(new_label)
@@ -878,14 +1140,14 @@ def save_labels(session, issue):
for label_name in delete_label_names:
session.query(Label).filter(Label.issue_id == existing_issue.id, Label.name == label_name).delete()
+
def save_event_info(session, event_dict):
'''
Save a dictionary of event into to the datastore session then return
that event instance
'''
# Select the current event, filtering on event_url and organization name.
- filter = Event.event_url == event_dict['event_url'], \
- Event.organization_name == event_dict['organization_name']
+ filter = Event.event_url == event_dict['event_url'], Event.organization_name == event_dict['organization_name']
existing_event = session.query(Event).filter(*filter).first()
# If this is a new event, save and return it.
@@ -902,14 +1164,16 @@ def save_event_info(session, event_dict):
for (field, value) in event_dict.items():
setattr(existing_event, field, value)
+ return existing_event
+
+
def save_story_info(session, story_dict):
'''
Save a dictionary of story into to the datastore session then return
that story instance
'''
# Select the current story, filtering on link and organization name.
- filter = Story.organization_name == story_dict['organization_name'], \
- Story.link == story_dict['link']
+ filter = Story.organization_name == story_dict['organization_name'], Story.link == story_dict['link']
existing_story = session.query(Story).filter(*filter).first()
@@ -927,7 +1191,16 @@ def save_story_info(session, story_dict):
for (field, value) in story_dict.items():
setattr(existing_story, field, value)
+ return existing_story
+
+
def get_event_group_identifier(events_url):
+ ''' Extract a group identifier from a meetup.com event URL
+ '''
+ if 'meetup.com' not in events_url:
+ logging.error("Only Meetup.com events work right now.")
+ return None
+
parse_result = urlparse(events_url)
url_parts = parse_result.path.split('/')
identifier = url_parts.pop()
@@ -939,50 +1212,60 @@ def get_event_group_identifier(events_url):
return None
-def get_attendance(peopledb, organization_url, organization_name):
- ''' Get the attendance of an org from the peopledb '''
-
- # Total attendance
- q = ''' SELECT COUNT(*) AS total FROM attendance
- WHERE organization_url = %s '''
- peopledb.execute(q,(organization_url,))
- total = int(peopledb.fetchone()["total"])
-
- # weekly attendance
- q = ''' SELECT COUNT(*) AS total,
- to_char(datetime, 'YYYY WW') AS week
- FROM attendance
- WHERE organization_url = %s
- GROUP BY week '''
- peopledb.execute(q,(organization_url,))
- weekly = peopledb.fetchall()
- weekly = { week["week"] : int(week["total"]) for week in weekly }
-
- attendance = {
- "organization_name" : organization_name,
- "organization_url" : organization_url,
- "total" : total,
- "weekly" : weekly
- }
-
- return attendance
-
-def update_attendance(db, organization_name, attendance):
- ''' Update exisiting attendance '''
+
+def update_attendance(session, organization_name, attendance_dict):
+ ''' Update exisiting attendance
+ '''
+ # Select the current attendance, filtering on organization
filter = Attendance.organization_name == organization_name
- existing_attendance = db.session.query(Attendance).filter(filter).first()
- if existing_attendance:
- existing_attendance.total = attendance["total"]
- existing_attendance.weekly = attendance["weekly"]
- db.session.add(existing_attendance)
- else:
- new_att = Attendance(**attendance)
- db.session.add(new_att)
- db.session.commit()
+ existing_attendance = session.query(Attendance).filter(filter).first()
+
+ # if this is a new attendance, save and return it
+ if not existing_attendance:
+ new_attendance = Attendance(**attendance_dict)
+ session.add(new_attendance)
+ return new_attendance
+
+ # Update existing attendance details
+ existing_attendance.total = attendance_dict["total"]
+ existing_attendance.weekly = attendance_dict["weekly"]
+
+ return existing_attendance
+
+
+def get_logo(org_info):
+ '''
+ get an organization's logo, looking first at 'logo_url' in the JSON and
+ then Github (project lists url)
+ '''
+ # allow specifying a logo_url in the json file
+ if 'logo_url' in org_info:
+ return org_info['logo_url']
+
+ if 'projects_list_url' not in org_info:
+ return None
+
+ github_username = parse_github_user(org_info['projects_list_url'])
+ if github_username:
+ # NOTE: This uses the /users/:id API endpoint to handle both cases
+ # where the brigade's profile is a single user account or an
+ # organizational account.
+ request_url = GITHUB_USER_API_URL.format(username=github_username)
+ got = get_github_api(request_url)
+ if got.status_code == 404:
+ logger.error("Got 404 for GitHub username " + github_username)
+ return
+
+ try:
+ github_response = got.json()
+ return github_response['avatar_url']
+ except ValueError:
+ logger.error("Malformed GitHub JSON fetching organization URL for " + github_username)
+ return
def main(org_name=None, org_sources=None):
- ''' Run update over all organizations. Optionally, update just one.
+ ''' Update the API's database
'''
# set org_sources
org_sources = org_sources or ORG_SOURCES_FILENAME
@@ -994,15 +1277,19 @@ def main(org_name=None, org_sources=None):
orgs_info = get_organizations(org_sources)
shuffle(orgs_info)
+ # Prioritize updating official CfA brigades' organizations first.
+ orgs_info.sort(cmp=lambda b1, b2: -1 if is_official_brigade(b1) else 0)
+
+ # If an organization name was passed, filter.
if org_name:
orgs_info = [org for org in orgs_info if org['name'] == org_name]
- # Iterate over organizations and projects, saving them to db.session.
+ # Retrieve and save all information about the organizations
for org_info in orgs_info:
if not is_safe_name(org_info['name']):
error_dict = {
- "error": unicode('ValueError: Bad organization name: "%s"' % org_info['name']),
+ "error": unicode('ValueError: Bad organization name: "{}"'.format(org_info['name'])),
"time": datetime.now()
}
new_error = Error(**error_dict)
@@ -1011,98 +1298,87 @@ def main(org_name=None, org_sources=None):
db.session.commit()
continue
- try:
- filter = Organization.name == org_info['name']
- existing_org = db.session.query(Organization).filter(filter).first()
+ # don't try to process orgs if we're throttled
+ if GITHUB_THROTTLING:
organization_names.add(org_info['name'])
+ continue
- # Mark everything associated with this organization for deletion at first.
+ try:
+ # Mark everything associated with this organization for deletion
# :::here (event/false, story/false, project/false, organization/false)
db.session.execute(db.update(Event, values={'keep': False}).where(Event.organization_name == org_info['name']))
db.session.execute(db.update(Story, values={'keep': False}).where(Story.organization_name == org_info['name']))
db.session.execute(db.update(Project, values={'keep': False}).where(Project.organization_name == org_info['name']))
db.session.execute(db.update(Organization, values={'keep': False}).where(Organization.name == org_info['name']))
- # commit the false keeps
- db.session.commit()
- # Empty lat longs are okay.
- if 'latitude' in org_info:
- if not org_info['latitude']:
- org_info['latitude'] = None
- if 'longitude' in org_info:
- if not org_info['longitude']:
- org_info['longitude'] = None
+ # ORGANIZATION INFO
+ # Save or update the organization
+ org_info.update({'logo_url': get_logo(org_info)})
organization = save_organization_info(db.session, org_info)
-
organization_names.add(organization.name)
- # flush the organization
- db.session.flush()
+ # commit the organization and the false keeps
+ db.session.commit()
+
+
+ # STORIES
if organization.rss or organization.website:
- logging.info("Gathering all of %s's stories." % organization.name)
+ logging.info(u"Gathering all of {}'s stories.".format(organization.name))
stories = get_stories(organization)
- if stories:
- for story_info in stories:
- save_story_info(db.session, story_info)
- # flush the stories
- db.session.flush()
+ # build and commit stories
+ for story_info in stories:
+ save_story_info(db.session, story_info)
+ db.session.commit()
+ # PROJECTS, ISSUES and LABELS
if organization.projects_list_url:
- logging.info("Gathering all of %s's projects." % organization.name)
+ logging.info(u"Gathering all of {}'s projects.".format(organization.name))
projects = get_projects(organization)
+ # build and commit projects
for proj_dict in projects:
- save_project_info(db.session, proj_dict)
- # flush the projects
- db.session.flush()
-
+ saved_project = save_project_info(db.session, proj_dict)
+ db.session.commit()
+
+ logging.info(u'Gathering all issues for this {} project: {}.'.format(organization.name, saved_project.name))
+ issues = get_issues(saved_project)
+ # build and commit issues and labels
+ for issue_dict in issues:
+ save_issue_info(db.session, issue_dict)
+ db.session.commit()
+ save_labels_info(db.session, issue_dict)
+ db.session.commit()
+
+ # EVENTS
if organization.events_url:
- if not meetup_key:
- logging.error("No Meetup.com key set.")
- if 'meetup.com' not in organization.events_url:
- logging.error("Only Meetup.com events work right now.")
+ logging.info(u"Gathering all of {}'s events.".format(organization.name))
+ identifier = get_event_group_identifier(organization.events_url)
+ if identifier:
+ # build and commit events
+ for event in get_meetup_events(organization, identifier):
+ save_event_info(db.session, event)
+ db.session.commit()
+
+ # Get and save the meetup.com member count for this organization
+ members = get_meetup_count(organization, identifier)
+ # Don't overwrite the old value if we got None back
+ if members:
+ organization.member_count = members
+ db.session.commit()
+
else:
- logging.info("Gathering all of %s's events." % organization.name)
- identifier = get_event_group_identifier(organization.events_url)
- if identifier:
- for event in get_meetup_events(organization, identifier):
- save_event_info(db.session, event)
- # flush the events
- db.session.flush()
- else:
- logging.error("%s does not have a valid events url" % organization.name)
-
- # Get issues for all of the projects
- logging.info("Gathering all of %s's open GitHub issues." % organization.name)
- issues = get_issues(organization.name)
- for issue in issues:
- save_issue(db.session, issue)
-
- # flush the issues
- db.session.flush()
- for issue in issues:
- save_labels(db.session, issue)
-
- # Get attendance data
- with connect(os.environ["PEOPLEDB"]) as conn:
- with conn.cursor(cursor_factory=extras.RealDictCursor) as peopledb:
- cfapi_url = "https://www.codeforamerica.org/api/organizations/"
- organization_url = cfapi_url + organization.api_id()
- attendance = get_attendance(peopledb, organization_url, organization.name)
-
- if attendance:
- update_attendance(db, organization.name, attendance)
-
- # commit everything
- db.session.commit()
+ logging.error(u'{} does not have a valid events url'.format(organization.name))
# Remove everything marked for deletion.
# :::here (event/delete, story/delete, project/delete, issue/delete, organization/delete)
- db.session.query(Event).filter(Event.keep == False).delete()
- db.session.query(Story).filter(Story.keep == False).delete()
- db.session.query(Issue).filter(Issue.keep == False).delete()
- db.session.query(Project).filter(Project.keep == False).delete()
- db.session.query(Organization).filter(Organization.keep == False).delete()
+ num_events = db.session.query(Event).filter(Event.keep == False).delete()
+ num_stories = db.session.query(Story).filter(Story.keep == False).delete()
+ num_issues = db.session.query(Issue).filter(Issue.keep == False).delete()
+ num_projects = db.session.query(Project).filter(Project.keep == False).delete()
+ num_orgs = db.session.query(Organization).filter(Organization.keep == False).delete()
+
+ logging.info(u'Deleted {} organizations, {} projects, {} issues, {} stories, {} events'.format(num_orgs, num_projects, num_issues, num_stories, num_events))
+
# commit objects deleted for keep=False
db.session.commit()
@@ -1126,11 +1402,22 @@ def main(org_name=None, org_sources=None):
# commit for deleting orphaned organizations
db.session.commit()
+
parser = ArgumentParser(description='''Update database from CSV source URL.''')
parser.add_argument('--name', dest='name', help='Single organization name to update.')
-parser.add_argument('--test', action='store_const', dest='org_sources', const=TEST_ORG_SOURCES_FILENAME, help='Use the testing list of organizations.')
+parser.add_argument('--sources', dest='sources', help='URL of an organization sources JSON file.')
+parser.add_argument('--test', action='store_const', dest='test_sources', const=TEST_ORG_SOURCES_FILENAME, help='Use the testing list of organizations.')
if __name__ == "__main__":
args = parser.parse_args()
org_name = args.name and args.name.decode('utf8') or ''
- main(org_name=org_name, org_sources=args.org_sources)
+ org_sources = args.sources and args.sources.decode('utf8') or ''
+ if args.test_sources and not org_sources:
+ org_sources = args.test_sources
+
+ try:
+ main(org_name=org_name, org_sources=org_sources)
+ except:
+ if SENTRY:
+ SENTRY.captureException()
+ raise
diff --git a/runtime.txt b/runtime.txt
new file mode 100644
index 0000000..f27f1cc
--- /dev/null
+++ b/runtime.txt
@@ -0,0 +1 @@
+python-2.7.15
diff --git a/scripts/civicjson_stats.py b/scripts/civicjson_stats.py
index e42c341..0fac59c 100644
--- a/scripts/civicjson_stats.py
+++ b/scripts/civicjson_stats.py
@@ -1,34 +1,35 @@
from requests import get
-from time import sleep
import json
-civicjson_urls = ["https://raw.githubusercontent.com/rasmi/my-neighborhood/master/civic.json",
-"https://raw.githubusercontent.com/BetaNYC/civic.json/master/civic.json",
-"https://raw.githubusercontent.com/ameensol/dataExplorer/master/civic.json",
-"https://raw.githubusercontent.com/ameensol/dataExplorerAPI/master/civic.json",
-"https://raw.githubusercontent.com/BetaNYC/betanyc-support-ribbon-css/master/civic.json",
-"https://raw.githubusercontent.com/BetaNYC/NY-Waterways-GTFS-data/master/civic.json",
-"https://raw.githubusercontent.com/rasmi/homeless-nyc/master/civic.json",
-"https://raw.githubusercontent.com/MTA-Service-Alerts-beta-nyc/service-alerts/master/civic.json",
-"https://raw.githubusercontent.com/seanluciotolentino/dangerous-intersections/master/civic.json",
-"https://raw.githubusercontent.com/clhenrick/am-i-rent-stabilized/master/civic.json",
-"https://raw.githubusercontent.com/josselinphilippe/bagitnyc/master/civic.json",
-"https://raw.githubusercontent.com/hondacivicapps/hondacivicapps.github.io/master/civic.json",
-"https://raw.githubusercontent.com/codefordc/guides/master/civic.json",
-"https://raw.githubusercontent.com/DangerousRDNYC/DangerousRDNYC/master/civic.json",
-"https://raw.githubusercontent.com/Emrals/Emrals-Android/master/civic.json",
-"https://raw.githubusercontent.com/codefordc/dc-campaign-finance-watch/master/civic.json",
-"https://raw.githubusercontent.com/rasmi/crime-nyc/master/civic.json",
-"https://raw.githubusercontent.com/codefordc/open211/master/civic.json",
-"https://raw.githubusercontent.com/codefordc/ancfinder/master/civic.json",
-"https://raw.githubusercontent.com/codefordc/codefordc-2.0/master/civic.json",
-"https://raw.githubusercontent.com/codefordc/districthousing/master/civic.json",
-"https://raw.githubusercontent.com/childcaremap/NYCdaycare/master/civic.json",
-"https://raw.githubusercontent.com/NYPDVisionZeroAccountability/compstat-vs-moving-violation-enforcement/master/civic.json",
-"https://raw.githubusercontent.com/BetaNYC/Bike-Share-Data-Best-Practices/master/civic.json",
-"https://raw.githubusercontent.com/BetaNYC/budgetBuddy/master/civic.json",
-"https://raw.githubusercontent.com/talos/acris-bigquery/master/civic.json",
-"https://raw.githubusercontent.com/camsys/onebusaway-nyc-atstop/master/civic.json"]
+civicjson_urls = [
+ "https://raw.githubusercontent.com/rasmi/my-neighborhood/master/civic.json",
+ "https://raw.githubusercontent.com/BetaNYC/civic.json/master/civic.json",
+ "https://raw.githubusercontent.com/ameensol/dataExplorer/master/civic.json",
+ "https://raw.githubusercontent.com/ameensol/dataExplorerAPI/master/civic.json",
+ "https://raw.githubusercontent.com/BetaNYC/betanyc-support-ribbon-css/master/civic.json",
+ "https://raw.githubusercontent.com/BetaNYC/NY-Waterways-GTFS-data/master/civic.json",
+ "https://raw.githubusercontent.com/rasmi/homeless-nyc/master/civic.json",
+ "https://raw.githubusercontent.com/MTA-Service-Alerts-beta-nyc/service-alerts/master/civic.json",
+ "https://raw.githubusercontent.com/seanluciotolentino/dangerous-intersections/master/civic.json",
+ "https://raw.githubusercontent.com/clhenrick/am-i-rent-stabilized/master/civic.json",
+ "https://raw.githubusercontent.com/josselinphilippe/bagitnyc/master/civic.json",
+ "https://raw.githubusercontent.com/hondacivicapps/hondacivicapps.github.io/master/civic.json",
+ "https://raw.githubusercontent.com/codefordc/guides/master/civic.json",
+ "https://raw.githubusercontent.com/DangerousRDNYC/DangerousRDNYC/master/civic.json",
+ "https://raw.githubusercontent.com/Emrals/Emrals-Android/master/civic.json",
+ "https://raw.githubusercontent.com/codefordc/dc-campaign-finance-watch/master/civic.json",
+ "https://raw.githubusercontent.com/rasmi/crime-nyc/master/civic.json",
+ "https://raw.githubusercontent.com/codefordc/open211/master/civic.json",
+ "https://raw.githubusercontent.com/codefordc/ancfinder/master/civic.json",
+ "https://raw.githubusercontent.com/codefordc/codefordc-2.0/master/civic.json",
+ "https://raw.githubusercontent.com/codefordc/districthousing/master/civic.json",
+ "https://raw.githubusercontent.com/childcaremap/NYCdaycare/master/civic.json",
+ "https://raw.githubusercontent.com/NYPDVisionZeroAccountability/compstat-vs-moving-violation-enforcement/master/civic.json",
+ "https://raw.githubusercontent.com/BetaNYC/Bike-Share-Data-Best-Practices/master/civic.json",
+ "https://raw.githubusercontent.com/BetaNYC/budgetBuddy/master/civic.json",
+ "https://raw.githubusercontent.com/talos/acris-bigquery/master/civic.json",
+ "https://raw.githubusercontent.com/camsys/onebusaway-nyc-atstop/master/civic.json"
+]
key_counts = {
# key : count
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..9cea7b6
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,7 @@
+[flake8]
+# Ignore:
+# E501 = line too long
+# E711 = comparison to None should be 'if cond is None:
+# E712 = comparison to True should be ‘if cond is True:’ or ‘if cond:’
+ignore = E501,E711,E712
+exclude = migrations,test
diff --git a/templates/index.html b/templates/index.html
index 716e099..b226f0c 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -2,16 +2,6 @@
-
-
-
-
-
-
Civic Tech Movement API
@@ -73,6 +63,9 @@ Url parameters
per_page (integer)
The number of results to return per page:
/api/organizations?per_page=5
+ tags(array of strings)
+ An array of tags to filter the results by. For example, to select only the Code for America Brigades:
+ /api/organizations?tags[]=Code%20for%20America&tags[]=Brigade
Organization properties
You can add any of the Organization properties as a parameter and the API will filter by organizations that have that property.
@@ -141,7 +134,7 @@
Retrieve a list of organizations, in
GeoJSON format
- for geographic applications..
+ for geographic applications.
Endpoint
@@ -340,10 +333,11 @@
Sample Response
"github_details": { … },
"organization": { … },
"organization_name": "Code for America",
- "tags": "community engagement, housing",
+ "tags": ["community engagement", "housing"],
"type": "web service",
"status": "In Progress",
- "issues": [ … ]
+ "issues": [ … ],
+ "commit_status": "success"
},
{
"id": 2,
@@ -376,10 +370,11 @@ Sample Response
"type": "Brigade, Code for All"
},
"organization_name": "Philly",
- "tags": "neighborhoods, news"
+ "tags": ["neighborhoods", "news"],
"type": "",
"status": "",
- "issues": [ … ]
+ "issues": [ … ],
+ "commit_status": "success"
},
{
"id": 2,
@@ -413,6 +408,8 @@
Numeric ID.
name
Name.
+ languages
+ A list of programming langauges used in the project
link_url
Homepage.
code_url
@@ -436,6 +433,8 @@
String with short description of project status.
issues
A list of project issues.
+ commit_status
+ The status of the latest commit. More info
@@ -453,7 +452,7 @@
Sample Response
"description": "A place-based call-in system for gathering and sharing community feedback",
"organization": { … },
"organization_name": "Code for America",
- "tags": "community engagement, housing",
+ "tags": ["community engagement", "housing"],
"type": "web service",
"github_details":
{
@@ -465,7 +464,6 @@ Sample Response
"forks_count": 18,
"homepage": "http://www.cityvoiceapp.com/",
"html_url": "https://github.com/codeforamerica/cityvoice",
- "language": "Ruby",
"name": "cityvoice",
"open_issues": 38,
"owner": { … },
@@ -493,6 +491,11 @@ Sample Response
"project_name": "cityvoice",
},
{ … }
+ ],
+ "languages" : [
+ "Python",
+ "CSS",
+ "HTML"
]
}
@@ -511,7 +514,7 @@ Endpoints
/api/events
/api/events/upcoming_events
- /api/events/past_events
+ /api/events/past_events
/api/organizations/{organization id}/events
/api/organizations/{organization id}/upcoming_events
/api/organizations/{organization id}/past_events
@@ -537,9 +540,9 @@
Response Properties
Event filters
upcoming_events
- Only returns events happening in the future.
+ Only returns events happening presently or in the future.
past_events
- Only returns events that have already happened.
+ Only returns events that have already ended.
Sample Request
@@ -778,7 +781,7 @@ Response Properties
Issue filters
labels (comma separated)
- Return issues that have atleast one of the given labels
+ Return issues that have at least one of the given labels
Sample Request
diff --git a/test/factories.py b/test/factories.py
index 990eb03..88cb201 100644
--- a/test/factories.py
+++ b/test/factories.py
@@ -33,10 +33,12 @@ class ProjectFactory(SQLAlchemyModelFactory):
description = u'This is a description'
type = factory.LazyAttribute(lambda n: choice([u'web service', u'api', u'data standard']))
categories = factory.LazyAttribute(lambda n: choice([u'housing', u'community engagement', u'criminal justice', u'education']))
- tags = factory.LazyAttribute(lambda n: choice([u'civic', u'mapping']))
+ tags = [u'what', u'ever', u'', u'†≈ç®åz¥≈†']
github_details = {'repo': u'git@github.com:codeforamerica/civic-project.git'}
organization_name = factory.LazyAttribute(lambda e: OrganizationFactory().name)
status = u'Project status'
+ languages = [u'Python', u'CSS']
+ last_updated = factory.LazyAttribute(lambda o: datetime.utcnow())
class EventFactory(SQLAlchemyModelFactory):
FACTORY_FOR = Event
@@ -49,11 +51,16 @@ class EventFactory(SQLAlchemyModelFactory):
location = u'155 9th St., San Francisco, CA'
now = factory.LazyAttribute(lambda o: datetime.utcnow())
- start_time_notz = factory.LazyAttribute(lambda o: o.now + timedelta(hours=10))
- end_time_notz = factory.LazyAttribute(lambda o: o.now + timedelta(hours=12))
- utc_offset = -28800
+ start_time_notz = factory.LazyAttribute(lambda o: o.now - timedelta(hours=10))
+ end_time_notz = factory.LazyAttribute(lambda o: o.start_time_notz + timedelta(hours=3))
+ utc_offset = -28800 # 8 hours
created_at = factory.LazyAttribute(lambda o: o.now)
organization_name = factory.LazyAttribute(lambda e: OrganizationFactory().name)
+ lat = 37.7749
+ lon = -122.4194
+
+ rsvps = 1234
+
class StoryFactory(SQLAlchemyModelFactory):
FACTORY_FOR = Story
@@ -71,6 +78,8 @@ class IssueFactory(SQLAlchemyModelFactory):
title = factory.Sequence(lambda n: u'Civic Issue {0}'.format(n))
html_url = factory.Sequence(lambda n: u'http://www.github.com/codeforamerica/cfapi/issues/{0}'.format(n))
body = factory.Sequence(lambda n: u'Civic Issue blah blah blah {0}'.format(n))
+ created_at = factory.LazyAttribute(lambda o: datetime.utcnow())
+ updated_at = factory.LazyAttribute(lambda o: datetime.utcnow())
project_id = factory.LazyAttribute(lambda e: ProjectFactory().id)
@@ -87,11 +96,11 @@ class AttendanceFactory(SQLAlchemyModelFactory):
FACTORY_SESSION = db.session
organization_name = factory.LazyAttribute(lambda e: OrganizationFactory().name)
- organization_url = "https://www.codeforamerica.org/api/organizations/" + str(factory.LazyAttribute(lambda e: OrganizationFactory().name)).replace(" ","-")
- total = randint(1,1000)
+ organization_url = "https://www.codeforamerica.org/api/organizations/" + str(factory.LazyAttribute(lambda e: OrganizationFactory().name)).replace(" ", "-")
+ total = randint(1, 1000)
weekly = {
- "2014 01" : randint(1,50),
- "2014 02" : randint(1,50),
- "2015 01" : randint(1,50),
- "2015 02" : randint(1,50)
+ "2014 01": randint(1, 50),
+ "2014 02": randint(1, 50),
+ "2015 01": randint(1, 50),
+ "2015 02": randint(1, 50)
}
diff --git a/test/integration/test_attendance.py b/test/integration/test_attendance.py
index f7f8d8d..b1827e8 100644
--- a/test/integration/test_attendance.py
+++ b/test/integration/test_attendance.py
@@ -8,12 +8,16 @@
class TestAttendance(IntegrationTest):
def test_attendance(self):
- cfsf = OrganizationFactory(name="Code for San Francisco")
- url = "https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
- cfsf_att = AttendanceFactory(organization_name="Code for San Francisco", organization_url=url)
- oakland = OrganizationFactory(name="Open Oakland")
- url = "https://www.codeforamerica.org/api/organizations/Open-Oakland"
- oakland_att = AttendanceFactory(organization_name="Open Oakland", organization_url=url)
+ cfsf = OrganizationFactory(name=u"Code for San Francisco")
+ url = u"https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
+ cfsf_att = AttendanceFactory(organization_name=u"Code for San Francisco", organization_url=url)
+ oakland = OrganizationFactory(name=u"Open Oakland")
+ url = u"https://www.codeforamerica.org/api/organizations/Open-Oakland"
+ oakland_att = AttendanceFactory(organization_name=u"Open Oakland", organization_url=url)
+ db.session.add(cfsf)
+ db.session.add(cfsf_att)
+ db.session.add(oakland)
+ db.session.add(oakland_att)
db.session.commit()
response = self.app.get('/api/attendance')
@@ -34,33 +38,31 @@ def test_attendance(self):
weekly[week] += att.weekly[week]
else:
weekly[week] = att.weekly[week]
- self.assertEqual(response["total"],total)
- self.assertEqual(response["weekly"],weekly)
-
+ self.assertEqual(response["total"], total)
+ self.assertEqual(response["weekly"], weekly)
def test_orgs_attendance(self):
- OrganizationFactory(name="Code for San Francisco")
- url = "https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
- AttendanceFactory(organization_name="Code for San Francisco", organization_url=url)
- OrganizationFactory(name="Open Oakland")
- url = "https://www.codeforamerica.org/api/organizations/Open-Oakland"
- AttendanceFactory(organization_name="Open Oakland", organization_url=url)
+ OrganizationFactory(name=u"Code for San Francisco")
+ url = u"https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
+ AttendanceFactory(organization_name=u"Code for San Francisco", organization_url=url)
+ OrganizationFactory(name=u"Open Oakland")
+ url = u"https://www.codeforamerica.org/api/organizations/Open-Oakland"
+ AttendanceFactory(organization_name=u"Open Oakland", organization_url=url)
db.session.commit()
response = self.app.get('/api/organizations/attendance')
self.assertEquals(response.status_code, 200)
response = json.loads(response.data)
- self.assertIsInstance(response, list)
- self.assertTrue("organization_name" in response[0].keys())
- self.assertTrue("cfapi_url" in response[0].keys())
- self.assertTrue("total" in response[0].keys())
- self.assertTrue("weekly" in response[0].keys())
-
+ self.assertIsInstance(response, dict)
+ self.assertTrue("organization_name" in response['organizations'][0].keys())
+ self.assertTrue("cfapi_url" in response['organizations'][0].keys())
+ self.assertTrue("total" in response['organizations'][0].keys())
+ self.assertTrue("weekly" in response['organizations'][0].keys())
def test_org_attendance(self):
- OrganizationFactory(name="Code for San Francisco")
- url = "https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
- AttendanceFactory(organization_name="Code for San Francisco", organization_url=url)
+ OrganizationFactory(name=u"Code for San Francisco")
+ url = u"https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
+ AttendanceFactory(organization_name=u"Code for San Francisco", organization_url=url)
db.session.commit()
response = self.app.get('/api/organizations/Code-for-San-Francisco/attendance')
@@ -71,4 +73,3 @@ def test_org_attendance(self):
self.assertTrue("cfapi_url" in response.keys())
self.assertTrue("total" in response.keys())
self.assertTrue("weekly" in response.keys())
-
diff --git a/test/integration/test_events.py b/test/integration/test_events.py
index 0ceb1a2..b28aba7 100644
--- a/test/integration/test_events.py
+++ b/test/integration/test_events.py
@@ -156,6 +156,7 @@ def test_events(self):
assert isinstance(response['objects'][0]['organization'], dict)
assert isinstance(response['objects'][0]['organization_name'], unicode)
assert isinstance(response['objects'][0]['start_time'], unicode)
+ assert isinstance(response['objects'][0]['rsvps'], int)
def test_past_events(self):
'''
@@ -218,3 +219,27 @@ def test_events_query_filter(self):
response = json.loads(response.data)
self.assertEqual(response['total'], 1)
self.assertEqual(response['objects'][0]['name'], u'Awesome event')
+
+
+ def test_rsvp_routes(self):
+ org = OrganizationFactory(name=u"Code for San Francisco")
+ another_org = OrganizationFactory(type=u'Code for All')
+ awesome_event = EventFactory(name=u'Awesome event')
+ sad_event = EventFactory(name=u'Sad event', description=u'sad stuff will happen')
+
+ awesome_event.organization = org
+ sad_event.organization = another_org
+
+ db.session.commit()
+
+ # Make sure total number rsvps is 2468
+ response = self.app.get('/api/events/rsvps')
+ response = json.loads(response.data)
+ self.assertEqual(response["total"], 2468)
+
+ # Make sure org number rsvps is 1234
+ response = self.app.get('/api/organizations/Code-for-San-Francisco/events/rsvps')
+ response = json.loads(response.data)
+ self.assertEqual(response["total"], 1234)
+
+
diff --git a/test/integration/test_issues.py b/test/integration/test_issues.py
index bbccd65..303591c 100644
--- a/test/integration/test_issues.py
+++ b/test/integration/test_issues.py
@@ -18,7 +18,7 @@ def test_issues(self):
project = ProjectFactory(organization_name=organization.name)
db.session.add(project)
db.session.commit()
- issue = IssueFactory(project_id=project.id, title=u'TEST ISSUE', body=u'TEST ISSUE BODY')
+ issue = IssueFactory(project_id=project.id, title=u'TEST ISSUE', body=u'TEST ISSUE BODY', created_at="2013-06-06T00:12:30Z", updated_at="2014-02-21T20:43:16Z")
db.session.add(issue)
db.session.commit()
@@ -29,6 +29,8 @@ def test_issues(self):
self.assertEqual(response['total'], 1)
self.assertEqual(response['objects'][0]['title'], u'TEST ISSUE')
self.assertEqual(response['objects'][0]['body'], u'TEST ISSUE BODY')
+ self.assertEqual(response['objects'][0]['created_at'], u'2013-06-06T00:12:30Z')
+ self.assertEqual(response['objects'][0]['updated_at'], u'2014-02-21T20:43:16Z')
# Check for linked issues in linked project
self.assertTrue('project' in response['objects'][0])
@@ -42,7 +44,6 @@ def test_issues(self):
self.assertEqual(response.status_code, 200)
response = json.loads(response.data)
self.assertTrue('project' in response)
- self.assertTrue('issues' not in response['project'])
def test_issues_with_labels(self):
'''
diff --git a/test/integration/test_organizations.py b/test/integration/test_organizations.py
index 403a2a0..d85eed7 100644
--- a/test/integration/test_organizations.py
+++ b/test/integration/test_organizations.py
@@ -1,3 +1,4 @@
+# -- coding: utf-8 --
import json
from datetime import datetime, timedelta
import time
@@ -53,18 +54,33 @@ def test_orgs_projects_order(self):
def test_current_events(self):
"""
- The three soonest upcoming events should be returned.
+ The two soonest upcoming events should be returned.
If there are no events in the future, no events will be returned
"""
# Assuming today is Christmas...
organization = OrganizationFactory(name=u'Collective of Ericas')
db.session.flush()
- # Create multiple events, some in the future, one in the past
- EventFactory(organization_name=organization.name, name=u'Christmas Eve', start_time_notz=datetime.now() - timedelta(1))
- EventFactory(organization_name=organization.name, name=u'New Years', start_time_notz=datetime.now() + timedelta(7))
- EventFactory(organization_name=organization.name, name=u'MLK Day', start_time_notz=datetime.now() + timedelta(25))
- EventFactory(organization_name=organization.name, name=u'Cesar Chavez Day', start_time_notz=datetime.now() + timedelta(37))
+ event_utc_offset = EventFactory.attributes()['utc_offset']
+ now = datetime.utcnow()
+ now_notz = now + timedelta(seconds=event_utc_offset)
+
+ # Create multiple events, some in the very near future, one in the very recent past
+ EventFactory(organization_name=organization.name,
+ name=u'Christmas Eve',
+ start_time_notz=now_notz - timedelta(hours=3),
+ end_time_notz=now_notz - timedelta(seconds=1))
+ EventFactory(organization_name=organization.name,
+ name=u'New Years',
+ start_time_notz=now_notz - timedelta(hours=2),
+ end_time_notz=now_notz + timedelta(seconds=1))
+ EventFactory(organization_name=organization.name,
+ name=u'MLK Day',
+ start_time_notz=now_notz + timedelta(days=7))
+ EventFactory(organization_name=organization.name,
+ name=u'Cesar Chavez Day',
+ start_time_notz=now_notz + timedelta(days=30))
+
db.session.commit()
response = self.app.get('/api/organizations/Collective%20of%20Ericas')
@@ -241,6 +257,25 @@ def test_org_search_existing_phrase(self):
self.assertEqual(response['total'], 1)
self.assertEqual(len(response['objects']), 1)
+ def test_org_search_escaped_phrase(self):
+ OrganizationFactory(
+ name=u'Cöde%%for \'Ameriça',
+ )
+ db.session.commit()
+ response = self.app.get('/api/organizations?q=\'Ameriça')
+ response = json.loads(response.data)
+ assert isinstance(response['total'], int)
+ assert isinstance(response['objects'], list)
+ self.assertEqual(response['total'], 1)
+ self.assertEqual(len(response['objects']), 1)
+
+ response = self.app.get('/api/organizations?q=Cöde%')
+ response = json.loads(response.data)
+ assert isinstance(response['total'], int)
+ assert isinstance(response['objects'], list)
+ self.assertEqual(response['total'], 1)
+ self.assertEqual(len(response['objects']), 1)
+
def test_org_search_existing_part_of_phrase(self):
OrganizationFactory(
name=u'Code for San Francisco',
@@ -319,8 +354,8 @@ def test_organization_query_filter(self):
'''
Test that organization query params work as expected.
'''
- OrganizationFactory(name=u'Brigade Organization', type=u'Brigade')
- OrganizationFactory(name=u'Bayamon Organization', type=u'Brigade', city=u'Bayamon, PR')
+ OrganizationFactory(name=u'Brigade Organization', type=u'Brigade', tags=['Brigade', 'Official'])
+ OrganizationFactory(name=u'Bayamon Organization', type=u'Brigade', city=u'Bayamon, PR', tags=['Brigade'])
OrganizationFactory(name=u'Meetup Organization', type=u'Meetup')
db.session.commit()
@@ -343,6 +378,68 @@ def test_organization_query_filter(self):
response = json.loads(response.data)
self.assertEqual(response['total'], 0)
+ # Test tag-based filtering:
+ response = self.app.get('/api/organizations?tags[]=Brigade')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 2)
+ self.assertEqual(response['objects'][0]['name'], u'Brigade Organization')
+ self.assertEqual(response['objects'][1]['name'], u'Bayamon Organization')
+
+ response = self.app.get('/api/organizations?tags[]=Brigade&tags[]=Official')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 1)
+ self.assertEqual(response['objects'][0]['name'], u'Brigade Organization')
+
+ def test_organization_query_filter_with_unescaped_characters(self):
+ ''' Test that organization query params with unescaped characters work as expected.
+ '''
+ OrganizationFactory(name=u'Code for Addis Ababa', type=u'Code for All', city=u'Addis Ababa')
+ OrganizationFactory(name=u'Code for Ponta Grossa', type=u'Code for All', city=u'Ponta Grossa, PR')
+ OrganizationFactory(name=u'USDS', type=u'Government', city=u'Washington, DC')
+
+ db.session.commit()
+
+ response = self.app.get('/api/organizations?type=Code%20for%20All')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 2)
+ self.assertEqual(response['objects'][0]['name'], u'Code for Addis Ababa')
+ self.assertEqual(response['objects'][1]['name'], u'Code for Ponta Grossa')
+
+ response = self.app.get('/api/organizations?type=Code+for+All')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 2)
+ self.assertEqual(response['objects'][0]['name'], u'Code for Addis Ababa')
+ self.assertEqual(response['objects'][1]['name'], u'Code for Ponta Grossa')
+
+ response = self.app.get('/api/organizations?type=Code%2Bfor%2BAll')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 2)
+ self.assertEqual(response['objects'][0]['name'], u'Code for Addis Ababa')
+ self.assertEqual(response['objects'][1]['name'], u'Code for Ponta Grossa')
+
+ response = self.app.get('/api/organizations?city=Ponta%20Grossa')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 1)
+ self.assertEqual(response['objects'][0]['name'], u'Code for Ponta Grossa')
+
+ response = self.app.get('/api/organizations?city=Addis+Ababa')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 1)
+ self.assertEqual(response['objects'][0]['name'], u'Code for Addis Ababa')
+
+ response = self.app.get('/api/organizations?city=Washington%2C%2BDC')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 1)
+ self.assertEqual(response['objects'][0]['name'], u'USDS')
+
def test_organization_issues(self):
''' Test getting all of an organization's issues
'''
@@ -552,5 +649,28 @@ def test_org_dont_show_issues(self):
response = json.loads(response.data)
for org in response['objects']:
if org['current_projects']:
- self.assertFalse('issues' in org['current_projects'][0])
+ self.assertFalse(isinstance(org['current_projects'][0]["issues"], list))
break
+
+ def test_geojson(self):
+ ''' Test that /organization.geojson works '''
+
+ organization = OrganizationFactory()
+ org2 = OrganizationFactory()
+ del org2.latitude
+ del org2.longitude
+
+ db.session.flush()
+
+ response = self.app.get('/api/organizations.geojson')
+ response = json.loads(response.data)
+
+ # Test that features have expected attributes
+ org = response['features'][0]
+ self.assertTrue('geometry' in org.keys())
+ self.assertTrue('coordinates' in org['geometry'])
+ self.assertTrue('properties' in org.keys())
+ self.assertTrue('id' in org.keys())
+
+ # Test that only orgs with location data showed up
+ self.assertEqual(len(response['features']),1)
diff --git a/test/integration/test_projects.py b/test/integration/test_projects.py
index d10b494..9ac3e6b 100644
--- a/test/integration/test_projects.py
+++ b/test/integration/test_projects.py
@@ -1,3 +1,4 @@
+# -- coding: utf-8 --
import json
from datetime import datetime, timedelta
@@ -9,9 +10,9 @@
class TestProjects(IntegrationTest):
def test_all_projects_order(self):
- """
+ '''
Test that projects gets returned in order of last_updated
- """
+ '''
ProjectFactory(name=u'Project 1', last_updated='Mon, 01 Jan 2010 00:00:00 GMT')
ProjectFactory(name=u'Project 2', last_updated='Tue, 01 Jan 2011 00:00:00 GMT')
ProjectFactory(name=u'Non Github Project', last_updated='Wed, 01 Jan 2013 00:00:00', github_details=None)
@@ -37,7 +38,7 @@ def test_projects(self):
assert isinstance(response['total'], int)
assert isinstance(response['objects'], list)
assert isinstance(response['objects'][0]['categories'], unicode)
- assert isinstance(response['objects'][0]['tags'], unicode)
+ assert isinstance(response['objects'][0]['tags'], list)
assert isinstance(response['objects'][0]['code_url'], unicode)
assert isinstance(response['objects'][0]['description'], unicode)
assert isinstance(response['objects'][0]['github_details'], dict)
@@ -49,6 +50,7 @@ def test_projects(self):
assert isinstance(response['objects'][0]['organization_name'], unicode)
assert isinstance(response['objects'][0]['type'], unicode)
assert isinstance(response['objects'][0]['status'], unicode)
+ assert isinstance(response['objects'][0]['languages'], list)
def test_project_search_nonexisting_text(self):
''' Searching for non-existing text in the project and org/project
@@ -93,6 +95,42 @@ def test_project_search_existing_text(self):
self.assertEqual(org_project_response['total'], 1)
self.assertEqual(len(org_project_response['objects']), 1)
+ def test_project_search_escaped_text(self):
+ ''' Searching for escaped text in the project and org/project endpoints
+ returns expected results
+ '''
+ organization = OrganizationFactory(name=u"Code for San Francisco")
+ ProjectFactory(organization_name=organization.name, description=u'What\'s My \'District')
+ ProjectFactory(organization_name=organization.name, description=u'Cöde%%for%%Ameriça')
+ db.session.commit()
+ project_response = self.app.get('/api/projects?q=What\'s My \'District')
+ project_response = json.loads(project_response.data)
+ assert isinstance(project_response['total'], int)
+ assert isinstance(project_response['objects'], list)
+ self.assertEqual(project_response['total'], 1)
+ self.assertEqual(len(project_response['objects']), 1)
+
+ org_project_response = self.app.get("/api/organizations/Code-for-San-Francisco/projects?q='District")
+ org_project_response = json.loads(org_project_response.data)
+ assert isinstance(org_project_response['total'], int)
+ assert isinstance(org_project_response['objects'], list)
+ self.assertEqual(org_project_response['total'], 1)
+ self.assertEqual(len(org_project_response['objects']), 1)
+
+ project_response = self.app.get('/api/projects?q=%Ameriça')
+ project_response = json.loads(project_response.data)
+ assert isinstance(project_response['total'], int)
+ assert isinstance(project_response['objects'], list)
+ self.assertEqual(project_response['total'], 1)
+ self.assertEqual(len(project_response['objects']), 1)
+
+ org_project_response = self.app.get("/api/organizations/Code-for-San-Francisco/projects?q=Cöde%")
+ org_project_response = json.loads(org_project_response.data)
+ assert isinstance(org_project_response['total'], int)
+ assert isinstance(org_project_response['objects'], list)
+ self.assertEqual(org_project_response['total'], 1)
+ self.assertEqual(len(org_project_response['objects']), 1)
+
def test_project_search_existing_phrase(self):
''' Searching for an existing phrase in the project and org/project endpoints
returns expected results
@@ -268,6 +306,23 @@ def test_project_search_order_by_last_updated_sort_asc(self):
self.assertEqual(len(org_project_response["objects"]), 2)
self.assertEqual(org_project_response['objects'][0]['description'], 'ruby ruby ruby ruby ruby')
+ def test_project_search_ranked_order(self):
+ ''' Search results from the project and org/project endpoints are returned
+ with correct ranking values
+ '''
+ organization = OrganizationFactory(name=u"Code for San Francisco")
+ ProjectFactory(organization_name=organization.name, status=u'TEST', last_updated=datetime.now() - timedelta(10000))
+ ProjectFactory(organization_name=organization.name, description=u'testing a new thing', last_updated=datetime.now() - timedelta(1))
+ ProjectFactory(organization_name=organization.name, tags=[u'test,tags,what,ever'], last_updated=datetime.now() - timedelta(100))
+ ProjectFactory(organization_name=organization.name, last_updated=datetime.now())
+ db.session.commit()
+ project_response = self.app.get('/api/projects?q=TEST')
+ project_response = json.loads(project_response.data)
+ self.assertEqual(project_response['total'], 3)
+ self.assertEqual(project_response['objects'][0]['status'], u'TEST')
+ self.assertEqual(project_response['objects'][1]['tags'], [u'test,tags,what,ever'])
+ self.assertEqual(project_response['objects'][2]['description'], u'testing a new thing')
+
def test_project_return_only_ids(self):
''' Search results from the project and org/project endpoints are returned
as only IDs if requested
@@ -381,114 +436,140 @@ def test_project_search_includes_name(self):
self.assertEqual(len(org_project_response['objects']), 1)
self.assertEqual(org_project_response['objects'][0]['name'], 'My Cool Project')
- def test_project_search_includes_type(self):
- ''' The type field is included in search results from the project and org/project endpoints
- '''
- organization = OrganizationFactory(name=u"Code for San Francisco")
- ProjectFactory(organization_name=organization.name, type=u'mobile app')
- ProjectFactory(organization_name=organization.name, type=u'data portal')
- db.session.commit()
- project_response = self.app.get('/api/projects?q=portal')
- project_response = json.loads(project_response.data)
- self.assertEqual(len(project_response['objects']), 1)
- self.assertEqual(project_response['objects'][0]['type'], 'data portal')
-
- org_project_response = self.app.get('/api/organizations/Code-for-San-Francisco/projects?q=portal')
- org_project_response = json.loads(org_project_response.data)
- self.assertEqual(len(org_project_response['objects']), 1)
- self.assertEqual(org_project_response['objects'][0]['type'], 'data portal')
-
- def test_project_search_includes_categories(self):
- ''' The categories field is included in search results from the project and org/project endpoints
- '''
- organization = OrganizationFactory(name=u"Code for San Francisco")
- ProjectFactory(organization_name=organization.name, categories=u'project management, civic hacking')
- ProjectFactory(organization_name=organization.name, categories=u'animal control, twitter')
- db.session.commit()
- project_response = self.app.get('/api/projects?q=control')
- project_response = json.loads(project_response.data)
- self.assertEqual(len(project_response['objects']), 1)
- self.assertEqual(project_response['objects'][0]['categories'], 'animal control, twitter')
-
- org_project_response = self.app.get('/api/organizations/Code-for-San-Francisco/projects?q=control')
- org_project_response = json.loads(org_project_response.data)
- self.assertEqual(len(org_project_response['objects']), 1)
- self.assertEqual(org_project_response['objects'][0]['categories'], 'animal control, twitter')
-
def test_project_search_includes_tags(self):
- """
+ '''
The tags field is included in search results from the project and org/project endpoints
- """
+ '''
organization = OrganizationFactory(name=u"Code for San Francisco")
- ProjectFactory(organization_name=organization.name, tags=u'mapping, philly')
- ProjectFactory(organization_name=organization.name, tags=u'food stamps, health')
+ ProjectFactory(organization_name=organization.name, tags=['mapping', 'philly'])
+ ProjectFactory(organization_name=organization.name, tags=['food stamps', 'health'])
db.session.commit()
project_response = self.app.get('/api/projects?q=stamps')
project_response = json.loads(project_response.data)
self.assertEqual(len(project_response['objects']), 1)
- self.assertEqual(project_response['objects'][0]['tags'], 'food stamps, health')
+ self.assertEqual(project_response['objects'][0]['tags'], ['food stamps', 'health'])
org_project_response = self.app.get('/api/organizations/Code-for-San-Francisco/projects?q=stamps')
org_project_response = json.loads(org_project_response.data)
self.assertEqual(len(org_project_response['objects']), 1)
- self.assertEqual(org_project_response['objects'][0]['tags'], 'food stamps, health')
+ self.assertEqual(org_project_response['objects'][0]['tags'], ['food stamps', 'health'])
- def test_project_search_includes_github_details(self):
- ''' The github_details field is included in search results from the project and org/project endpoints
+ def test_project_search_includes_organization_name(self):
+ '''
+ The organization name is included in the project search
'''
organization = OrganizationFactory(name=u"Code for San Francisco")
- ProjectFactory(organization_name=organization.name, github_details=json.dumps({'panic': 'disco'}))
- ProjectFactory(organization_name=organization.name, github_details=json.dumps({'button': 'red'}))
+ ProjectFactory(organization_name=organization.name, name=u"Project One")
+ ProjectFactory(organization_name=organization.name, name=u"Project Two", description=u"America")
+
+ organization = OrganizationFactory(name=u"Code for America")
+ ProjectFactory(organization_name=organization.name, name=u"Project Three")
+ ProjectFactory(organization_name=organization.name, name=u"Project Four", tags=u"San Francisco")
db.session.commit()
- project_response = self.app.get('/api/projects?q=disco')
- project_response = json.loads(project_response.data)
- self.assertEqual(len(project_response['objects']), 1)
- self.assertEqual(project_response['objects'][0]['github_details'], '{"panic": "disco"}')
- org_project_response = self.app.get('/api/organizations/Code-for-San-Francisco/projects?q=disco')
- org_project_response = json.loads(org_project_response.data)
- self.assertEqual(len(org_project_response['objects']), 1)
- self.assertEqual(org_project_response['objects'][0]['github_details'], '{"panic": "disco"}')
+ # Test that org_name matches return before project name
+ project_response = self.app.get('/api/projects?q=Code+for+San+Francisco')
+ project_response = json.loads(project_response.data)
+ self.assertEqual(len(project_response['objects']), 3)
+ self.assertEqual(project_response['objects'][0]['name'], u'Project One')
+ self.assertEqual(project_response['objects'][1]['name'], u'Project Two')
+ self.assertEqual(project_response['objects'][2]['name'], u'Project Four')
+ self.assertTrue('San Francisco' in project_response['objects'][2]['tags'])
+
+ # Test that org name matches return before project description
+ project_response = self.app.get('/api/projects?q=Code for America')
+ project_response = json.loads(project_response.data)
+ self.assertEqual(len(project_response['objects']), 3)
+ self.assertEqual(project_response['objects'][0]['name'], u'Project Three')
+ self.assertEqual(project_response['objects'][1]['name'], u'Project Four')
+ self.assertEqual(project_response['objects'][2]['name'], u'Project Two')
+ self.assertEqual(project_response['objects'][2]['description'], u'America')
- def test_project_query_filter(self):
+ def test_project_organzation_type_filter(self):
'''
- Test that project query params work as expected.
+ Test searching for projects from certain types of organizations.
'''
- brigade = OrganizationFactory(name=u'Whatever', type=u'Brigade')
- brigade_somewhere_far = OrganizationFactory(name=u'Brigade Organization', type=u'Brigade, Code for All')
- web_project = ProjectFactory(name=u'Random Web App', type=u'web service')
- other_web_project = ProjectFactory(name=u'Random Web App 2', type=u'web service', description=u'Another')
- non_web_project = ProjectFactory(name=u'Random Other App', type=u'other service')
-
- web_project.organization = brigade
- non_web_project.organization = brigade_somewhere_far
-
- db.session.add(web_project)
- db.session.add(non_web_project)
+ brigade = OrganizationFactory(name=u'Brigade Org', type=u'Brigade, midwest')
+ code_for_all = OrganizationFactory(name=u'Code for All Org', type=u'Code for All')
+ gov_org = OrganizationFactory(name=u'Gov Org', type=u'Government')
+
+ brigade_project = ProjectFactory(name=u'Today Brigade project', organization_name=brigade.name)
+ code_for_all_project = ProjectFactory(name=u'Yesterday Code for All project', organization_name=code_for_all.name, last_updated=datetime.now() - timedelta(days=1))
+ gov_project = ProjectFactory(name=u'Two days ago Gov project', organization_name=gov_org.name, last_updated=datetime.now() - timedelta(days=2))
+ brigade_project2 = ProjectFactory(name=u'Three days ago Brigade project', organization_name=brigade.name, last_updated=datetime.now() - timedelta(days=3))
+ code_for_all_project2 = ProjectFactory(name=u'Four days ago Code for All project', organization_name=code_for_all.name, last_updated=datetime.now() - timedelta(days=4))
+ gov_project2 = ProjectFactory(name=u'Five days ago Gov project', organization_name=gov_org.name, last_updated=datetime.now() - timedelta(days=5))
+
+ db.session.add(brigade_project)
+ db.session.add(code_for_all_project)
+ db.session.add(gov_project)
+ db.session.add(brigade_project2)
+ db.session.add(code_for_all_project2)
+ db.session.add(gov_project2)
db.session.commit()
- response = self.app.get('/api/projects?type=web%20service')
+ # Test they return in order of last_updated
+ response = self.app.get('/api/projects')
self.assertEqual(response.status_code, 200)
response = json.loads(response.data)
- self.assertEqual(response['total'], 2)
- self.assertEqual(response['objects'][0]['name'], u'Random Web App')
- self.assertEqual(response['objects'][1]['name'], u'Random Web App 2')
-
- response = self.app.get('/api/projects?type=web%20service&description=Another')
+ self.assertEqual(response['total'], 6)
+ self.assertEqual(response['objects'][0]['name'], 'Today Brigade project')
+ self.assertEqual(response['objects'][1]['name'], 'Yesterday Code for All project')
+ self.assertEqual(response['objects'][2]['name'], 'Two days ago Gov project')
+ self.assertEqual(response['objects'][3]['name'], 'Three days ago Brigade project')
+ self.assertEqual(response['objects'][4]['name'], 'Four days ago Code for All project')
+ self.assertEqual(response['objects'][5]['name'], 'Five days ago Gov project')
+
+ # Test they return in order of last_updated, no matter the search order
+ response = self.app.get('/api/projects?organization_type=Government,Code+for+All,Brigade')
self.assertEqual(response.status_code, 200)
response = json.loads(response.data)
- self.assertEqual(response['total'], 1)
- self.assertEqual(response['objects'][0]['name'], u'Random Web App 2')
-
- response = self.app.get('/api/projects?type=different%20service')
+ self.assertEqual(response['total'], 6)
+ self.assertEqual(response['objects'][0]['name'], 'Today Brigade project')
+ self.assertEqual(response['objects'][1]['name'], 'Yesterday Code for All project')
+ self.assertEqual(response['objects'][2]['name'], 'Two days ago Gov project')
+ self.assertEqual(response['objects'][3]['name'], 'Three days ago Brigade project')
+ self.assertEqual(response['objects'][4]['name'], 'Four days ago Code for All project')
+ self.assertEqual(response['objects'][5]['name'], 'Five days ago Gov project')
+
+ response = self.app.get('/api/projects?organization_type=Brigade,Code+for+All')
self.assertEqual(response.status_code, 200)
response = json.loads(response.data)
- self.assertEqual(response['total'], 0)
+ self.assertEqual(response['total'], 4)
+ self.assertEqual(response['objects'][0]['name'], 'Today Brigade project')
+ self.assertEqual(response['objects'][1]['name'], 'Yesterday Code for All project')
+ self.assertEqual(response['objects'][2]['name'], 'Three days ago Brigade project')
+ self.assertEqual(response['objects'][3]['name'], 'Four days ago Code for All project')
+
+ # # Different order, same results
+ response = self.app.get('/api/projects?organization_type=Code+for+All,Brigade')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 4)
+ self.assertEqual(response['objects'][0]['name'], 'Today Brigade project')
+ self.assertEqual(response['objects'][1]['name'], 'Yesterday Code for All project')
+ self.assertEqual(response['objects'][2]['name'], 'Three days ago Brigade project')
+ self.assertEqual(response['objects'][3]['name'], 'Four days ago Code for All project')
- response = self.app.get('/api/projects?organization_type=Code+for+All')
+ response = self.app.get('/api/projects?organization_type=Code+for+All,Government')
+ self.assertEqual(response.status_code, 200)
+ response = json.loads(response.data)
+ self.assertEqual(response['total'], 4)
+ self.assertEqual(response['objects'][0]['name'], 'Yesterday Code for All project')
+ self.assertEqual(response['objects'][1]['name'], 'Two days ago Gov project')
+ self.assertEqual(response['objects'][2]['name'], 'Four days ago Code for All project')
+ self.assertEqual(response['objects'][3]['name'], 'Five days ago Gov project')
+
+ # # Different order, same results
+ response = self.app.get('/api/projects?organization_type=Government,Code+for+All')
self.assertEqual(response.status_code, 200)
response = json.loads(response.data)
- self.assertEqual(response['total'], 1)
+ self.assertEqual(response['total'], 4)
+ self.assertEqual(response['objects'][0]['name'], 'Yesterday Code for All project')
+ self.assertEqual(response['objects'][1]['name'], 'Two days ago Gov project')
+ self.assertEqual(response['objects'][2]['name'], 'Four days ago Code for All project')
+ self.assertEqual(response['objects'][3]['name'], 'Five days ago Gov project')
+
def test_project_cascading_deletes(self):
''' Test that issues get deleted when their parent
@@ -523,6 +604,9 @@ def test_project_cascading_deletes(self):
issue = IssueFactory(title=u'TEST ISSUE', project_id=project.id)
another_issue = IssueFactory(title=u'ANOTHER TEST ISSUE', project_id=project.id)
a_third_issue = IssueFactory(title=u'A THIRD TEST ISSUE', project_id=project.id)
+ db.session.add(issue)
+ db.session.add(another_issue)
+ db.session.add(a_third_issue)
db.session.commit()
# make sure the issues are in the db
@@ -533,3 +617,23 @@ def test_project_cascading_deletes(self):
db.session.commit()
issues = db.session.query(Issue).all()
self.assertFalse(len(issues))
+
+ def test_include_issues(self):
+ """ Test the include_issues flag """
+ project = ProjectFactory()
+ db.session.commit()
+ IssueFactory(project_id=project.id)
+ db.session.commit()
+
+ got = self.app.get("/api/projects?include_issues=True")
+ project = json.loads(got.data)['objects'][0]
+ self.assertTrue(isinstance(project['issues'], list))
+ got = self.app.get("/api/projects?include_issues=False")
+ project = json.loads(got.data)['objects'][0]
+ self.assertFalse(isinstance(project['issues'], list))
+ self.assertEqual("http://localhost/api/projects/1/issues", project["issues"])
+ got = self.app.get("/api/projects")
+ project = json.loads(got.data)['objects'][0]
+ self.assertFalse(isinstance(project['issues'], list))
+ self.assertEqual("http://localhost/api/projects/1/issues", project["issues"])
+
diff --git a/test/peopledbtest.pgsql b/test/peopledbtest.pgsql
index 1a958c2..aa2ac84 100644
--- a/test/peopledbtest.pgsql
+++ b/test/peopledbtest.pgsql
@@ -11,5 +11,5 @@ CREATE TABLE attendance
event_name TEXT,
-- Sometimes people answer informational questions when they check in.
- extras JSON
+ extras TEXT
);
\ No newline at end of file
diff --git a/test/updater/__init__.py b/test/updater/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/run_update_test.py b/test/updater/test_run_update.py
similarity index 71%
rename from run_update_test.py
rename to test/updater/test_run_update.py
index 669d6c3..31a6ea9 100644
--- a/run_update_test.py
+++ b/test/updater/test_run_update.py
@@ -5,7 +5,6 @@
import datetime
import logging
import time
-import json
from re import match, search, sub
from httmock import response, HTTMock
@@ -13,6 +12,12 @@
from psycopg2 import connect, extras
+from freezegun import freeze_time
+
+from csv import DictReader
+from StringIO import StringIO
+import json
+
root_logger = logging.getLogger()
root_logger.disabled = True
@@ -39,24 +44,13 @@ def setUp(self):
self.db.create_all()
import run_update
- run_update.github_throttling = False
+ run_update.GITHUB_THROTTLING = False
- # FAKE PEOPLEDB
- with connect('postgres:///peopledbtest') as conn:
- with conn.cursor() as db:
- with open('test/peopledbtest-destroy.pgsql') as filename:
- db.execute(filename.read())
- with open('test/peopledbtest.pgsql') as filename:
- db.execute(filename.read())
def tearDown(self):
self.db.session.close()
self.db.drop_all()
- with connect('postgres:///peopledbtest') as conn:
- with conn.cursor() as db:
- with open('test/peopledbtest-destroy.pgsql') as filename:
- db.execute(filename.read())
def setup_mock_rss_response(self):
''' This overwrites urllib2.urlopen to return a mock response, which stops
@@ -73,7 +67,7 @@ def setup_mock_rss_response(self):
urllib2.urlopen.return_value.read = Mock(return_value=rss_content)
return urllib2.urlopen
- def get_raw_organization_list(self, count=3):
+ def get_csv_organization_list(self, count=3):
if type(count) is not int:
count = 3
# 'https://github.com/codeforamerica' and 'https://www.github.com/orgs/codeforamerica' are transformed
@@ -81,24 +75,41 @@ def get_raw_organization_list(self, count=3):
lines = [u'''name,website,events_url,rss,projects_list_url'''.encode('utf8'), u'''Cöde for Ameriça,http://codeforamerica.org,http://www.meetup.com/events/Code-For-Charlotte/,http://www.codeforamerica.org/blog/feed/,http://example.com/cfa-projects.csv'''.encode('utf8'), u'''Code for America (2),,,,https://github.com/codeforamerica'''.encode('utf8'), u'''Code for America (3),,http://www.meetup.com/events/Code-For-Rhode-Island/,http://www.codeforamerica.org/blog/another/feed/,https://www.github.com/orgs/codeforamerica'''.encode('utf8')]
return '\n'.join(lines[0:count + 1])
+ def get_json_organization_list(self, count=3):
+ ''' Get the json version of the organization list
+ '''
+ raw_csv = self.get_csv_organization_list(count)
+ return json.dumps([item for item in DictReader(StringIO(raw_csv))])
+
def response_content(self, url, request):
# csv file of project descriptions
if url.geturl() == 'http://example.com/cfa-projects.csv':
- project_lines = ['''Name,description,link_url,code_url,type,categories,tags,status''', ''',,,https://github.com/codeforamerica/cityvoice,,,"safety, police, poverty",Shuttered''', ''',,,https://github.com/codeforamerica/bizfriendly-web,,,,''']
+ project_lines = ['''Name,description,link_url,code_url,type,categories,tags,status''', ''',,,https://github.com/codeforamerica/cityvoice,,,"safety, police, poverty",Shuttered''', ''',,,https://github.com/codeforamerica/bizfriendly-web/,,,"what,ever,,†≈ç®åz¥≈†",''']
if self.results_state == 'before':
return response(200, '''\n'''.join(project_lines[0:3]), {'content-type': 'text/csv; charset=UTF-8'})
elif self.results_state == 'after':
return response(200, '''\n'''.join(project_lines[0:2]), {'content-type': 'text/csv; charset=UTF-8'})
+ # json of user description
+ elif url.geturl() == 'https://api.github.com/users/codeforamerica':
+ return response(200, '''{ "login": "codeforamerica", "id": 337792, "avatar_url": "https://avatars2.githubusercontent.com/u/337792?v=4", "gravatar_id": "", "url": "https://api.github.com/users/codeforamerica", "html_url": "https://github.com/codeforamerica", "followers_url": "https://api.github.com/users/codeforamerica/followers", "following_url": "https://api.github.com/users/codeforamerica/following{/other_user}", "gists_url": "https://api.github.com/users/codeforamerica/gists{/gist_id}", "starred_url": "https://api.github.com/users/codeforamerica/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/codeforamerica/subscriptions", "organizations_url": "https://api.github.com/users/codeforamerica/orgs", "repos_url": "https://api.github.com/users/codeforamerica/repos", "events_url": "https://api.github.com/users/codeforamerica/events{/privacy}", "received_events_url": "https://api.github.com/users/codeforamerica/received_events", "type": "Organization", "site_admin": false, "name": "Code for America", "company": null, "blog": "http://codeforamerica.org", "location": null, "email": "labs@codeforamerica.org", "hireable": null, "bio": null, "public_repos": 659, "public_gists": 0, "followers": 0, "following": 0, "created_at": "2010-07-19T19:41:04Z", "updated_at": "2017-09-05T10:22:41Z" }''')
# json of project descriptions
elif url.geturl() == 'https://api.github.com/users/codeforamerica/repos':
- return response(200, '''[{ "id": 10515516, "name": "cityvoice", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792", "html_url": "https://github.com/codeforamerica", "type": "Organization"}, "html_url": "https://github.com/codeforamerica/cityvoice", "description": "A place-based call-in system for gathering and sharing community feedback", "url": "https://api.github.com/repos/codeforamerica/cityvoice", "contributors_url": "https://api.github.com/repos/codeforamerica/cityvoice/contributors", "created_at": "2013-06-06T00:12:30Z", "updated_at": "2014-02-21T20:43:16Z", "pushed_at": "2014-02-21T20:43:16Z", "homepage": "http://www.cityvoiceapp.com/", "stargazers_count": 10, "watchers_count": 10, "language": "Ruby", "forks_count": 12, "open_issues": 37 }]''', headers=dict(Link='; rel="next", ; rel="last"'))
+ return response(200, '''[{ "id": 10515516, "name": "cityvoice", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792", "html_url": "https://github.com/codeforamerica", "type": "Organization"}, "html_url": "https://github.com/codeforamerica/cityvoice", "description": "A place-based call-in system for gathering and sharing community feedback", "url": "https://api.github.com/repos/codeforamerica/cityvoice", "contributors_url": "https://api.github.com/repos/codeforamerica/cityvoice/contributors", "created_at": "2013-06-06T00:12:30Z", "updated_at": "2014-02-21T20:43:16Z", "pushed_at": "2014-02-21T20:43:16Z", "homepage": "http://www.cityvoiceapp.com/", "stargazers_count": 10, "watchers_count": 10, "language": "Ruby", "forks_count": 12, "open_issues": 37, "languages_url": "https://api.github.com/repos/codeforamerica/cityvoice/languages" }]''', headers=dict(Link='; rel="next", ; rel="last"'))
- # csv file of organization descriptions
+ # mock of programming languages
+ elif url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice/languages':
+ return response(200, ''' { "Ruby": 178825, "HTML": 80191, "JavaScript": 16028, "CSS": 8579, "Shell": 219 }''')
+
+ # json file of organization descriptions
# this catches the request for the URL contained in run_update.TEST_ORG_SOURCES_FILENAME
+ elif url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, self.get_json_organization_list(self.organization_count))
+
+ # csv file of organization descriptions
elif "docs.google.com" in url:
- return response(200, self.get_raw_organization_list(self.organization_count))
+ return response(200, self.get_csv_organization_list(self.organization_count))
# contents of civic.json file in root directory for cityvoice
elif "cityvoice/contents/civic.json" in url.geturl():
@@ -118,11 +129,11 @@ def response_content(self, url, request):
# json of project description (cityvoice)
elif url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice':
- return response(200, '''{ "id": 10515516, "name": "cityvoice", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792", "html_url": "https://github.com/codeforamerica", "type": "Organization"}, "html_url": "https://github.com/codeforamerica/cityvoice", "description": "A place-based call-in system for gathering and sharing community feedback", "url": "https://api.github.com/repos/codeforamerica/cityvoice", "contributors_url": "https://api.github.com/repos/codeforamerica/cityvoice/contributors", "created_at": "2013-06-06T00:12:30Z", "updated_at": "2014-02-21T20:43:16Z", "pushed_at": "2014-02-21T20:43:16Z", "homepage": "http://www.cityvoiceapp.com/", "stargazers_count": 10, "watchers_count": 10, "language": "Ruby", "forks_count": 12, "open_issues": 37 }''', {'last-modified': datetime.datetime.strptime('Fri, 15 Nov 2013 00:08:07 GMT', "%a, %d %b %Y %H:%M:%S GMT")})
+ return response(200, '''{ "id": 10515516, "name": "cityvoice", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792", "html_url": "https://github.com/codeforamerica", "type": "Organization"}, "html_url": "https://github.com/codeforamerica/cityvoice", "description": "A place-based call-in system for gathering and sharing community feedback", "url": "https://api.github.com/repos/codeforamerica/cityvoice", "contributors_url": "https://api.github.com/repos/codeforamerica/cityvoice/contributors", "created_at": "2013-06-06T00:12:30Z", "updated_at": "2014-02-21T20:43:16Z", "pushed_at": "2014-02-21T20:43:16Z", "homepage": "http://www.cityvoiceapp.com/", "stargazers_count": 10, "watchers_count": 10, "language": "Ruby", "languages_url": "https://api.github.com/repos/codeforamerica/cityvoice/languages", "forks_count": 12, "open_issues": 37, "subscribers_count": 40, "default_branch" : "master" }''', {'last-modified': datetime.datetime.strptime('Fri, 15 Nov 2013 00:08:07 GMT', "%a, %d %b %Y %H:%M:%S GMT")})
# json of project description (bizfriendly-web)
elif url.geturl() == 'https://api.github.com/repos/codeforamerica/bizfriendly-web':
- return response(200, ''' { "id": 11137392, "name": "bizfriendly-web", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792?v=3", "html_url": "https://github.com/codeforamerica", "type": "Organization" }, "html_url": "https://github.com/codeforamerica/bizfriendly-web", "description": "An online service that teaches small business owners how to use the internet to better run their businesses.", "url": "https://api.github.com/repos/codeforamerica/bizfriendly-web", "contributors_url": "https://api.github.com/repos/codeforamerica/bizfriendly-web/contributors", "created_at": "2013-07-02T23:14:10Z", "updated_at": "2014-11-02T18:55:33Z", "pushed_at": "2014-10-14T21:55:04Z", "homepage": "http://bizfriend.ly", "stargazers_count": 17, "watchers_count": 17, "language": "JavaScript", "forks_count": 21, "open_issues": 31 } ''', {'last-modified': datetime.datetime.strptime('Fri, 15 Nov 2013 00:08:07 GMT', "%a, %d %b %Y %H:%M:%S GMT")})
+ return response(200, ''' { "id": 11137392, "name": "bizfriendly-web", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792?v=3", "html_url": "https://github.com/codeforamerica", "type": "Organization" }, "html_url": "https://github.com/codeforamerica/bizfriendly-web", "description": "An online service that teaches small business owners how to use the internet to better run their businesses.", "url": "https://api.github.com/repos/codeforamerica/bizfriendly-web", "contributors_url": "https://api.github.com/repos/codeforamerica/bizfriendly-web/contributors", "created_at": "2013-07-02T23:14:10Z", "updated_at": "2014-11-02T18:55:33Z", "pushed_at": "2014-10-14T21:55:04Z", "homepage": "http://bizfriend.ly", "stargazers_count": 17, "watchers_count": 17, "language": "JavaScript", "languages_url": "https://api.github.com/repos/codeforamerica/cityvoice/languages", "forks_count": 21, "open_issues": 31, "subscribers_count": 44 } ''', {'last-modified': datetime.datetime.strptime('Fri, 15 Nov 2013 00:08:07 GMT', "%a, %d %b %Y %H:%M:%S GMT")})
# json of project contributors (cityvoice)
elif url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice/contributors' or url.geturl() == 'https://api.github.com/repos/codeforamerica/bizfriendly-web/contributors':
@@ -135,8 +146,8 @@ def response_content(self, url, request):
# json of project issues (cityvoice, bizfriendly-web)
elif url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice/issues' or url.geturl() == 'https://api.github.com/repos/codeforamerica/bizfriendly-web/issues':
# build issues dynamically based on results_state value
- issue_lines = ['''{"html_url": "https://github.com/codeforamerica/cityvoice/issue/210","title": "Important cityvoice issue", "labels": [ xxx ], "body" : "WHATEVER"}''', '''{"html_url": "https://github.com/codeforamerica/cityvoice/issue/211","title": "More important cityvoice issue", "labels": [ xxx ], "body" : "WHATEVER"}''']
- label_lines = ['''{ "color" : "84b6eb", "name" : "enhancement", "url": "https://api.github.com/repos/codeforamerica/cityvoice/labels/enhancement"}''', '''{ "color" : "84b6eb", "name" : "question", "url": "https://api.github.com/repos/codeforamerica/cityvoice/labels/question"}''']
+ issue_lines = ['''{"html_url": "https://github.com/codeforamerica/cityvoice/issue/210","title": "Important cityvoice issue", "labels": [ xxx ],"created_at": "2015-09-16T05:45:20Z", "updated_at": "2015-10-22T17:26:02Z", "body" : "WHATEVER"}''', '''{"html_url": "https://github.com/codeforamerica/cityvoice/issue/211","title": "More important cityvoice issue", "labels": [ xxx ], "created_at" : "2015-10-26T01:13:03Z", "updated_at" : "2015-10-26T18:06:54Z", "body" : "WHATEVER"}''']
+ label_lines = ['''{ "color" : "84b6eb", "name" : "enhancement", "url": "https://api.github.com/repos/codeforamerica/cityvoice/labels/enhancement", "node_id": "AAAAAA="}''', '''{ "color" : "84b6eb", "name" : "question", "url": "https://api.github.com/repos/codeforamerica/cityvoice/labels/question", "node_id": "BBBBBBB="}''']
issue_lines_before = [sub('xxx', ','.join(label_lines[0:2]), issue_lines[0]), sub('xxx', ','.join(label_lines[0:2]), issue_lines[1])]
issue_lines_after = [sub('xxx', ','.join(label_lines[0:1]), issue_lines[0])]
response_etag = {'ETag': '8456bc53d4cf6b78779ded3408886f82'}
@@ -154,8 +165,16 @@ def response_content(self, url, request):
elif url.geturl() == 'https://api.github.com/user/337792/repos?page=2':
return response(200, '''[ ]''', headers=dict(Link='; rel="prev", ; rel="first"'))
+ # mock commit status
+ elif url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice/commits/master/status':
+ return response(200, '''{ "state" : "success" } ''')
+
+ # elif meetup member count
+ elif 'https://api.meetup.com/2/groups?group_urlname=' in url.geturl():
+ return response(200, ''' { "results" : [ { "members" : 100 } ] } ''')
+
# json of meetup events
- elif 'meetup.com' in url.geturl() and 'Code-For-Charlotte' in url.geturl():
+ elif 'https://api.meetup.com/2/events?status=past,upcoming&format=json&group_urlname=' in url.geturl() and 'Code-For-Charlotte' in url.geturl():
events_filename = 'meetup_events.json'
if self.results_state == 'after':
events_filename = 'meetup_events_fewer.json'
@@ -166,7 +185,7 @@ def response_content(self, url, request):
return response(200, events_content)
# json of alternate meetup events
- elif 'meetup.com' in url.geturl() and 'Code-For-Rhode-Island' in url.geturl():
+ elif 'https://api.meetup.com/2/events?status=past,upcoming&format=json&group_urlname=' in url.geturl() and 'Code-For-Rhode-Island' in url.geturl():
events_file = open('meetup_events_another.json')
events_content = events_file.read()
events_file.close()
@@ -232,6 +251,7 @@ def overwrite_response_content(url, request):
project = self.db.session.query(Project).filter(filter).first()
self.assertIsNotNone(project)
self.assertEqual(project.name, u'bizfriendly-web')
+ self.assertEqual(project.tags, [u'what', u'ever', u'', u'†≈ç®åz¥≈†'])
# check for the one project status
filter = [Project.organization_name == u'Cöde for Ameriça', Project.name == u'cityvoice']
@@ -264,16 +284,40 @@ def test_main_with_good_new_data(self):
old_project = ProjectFactory(name=u'Old Project', organization_name=u'Old Organization')
old_event = EventFactory(name=u'Old Event', organization_name=u'Old Organization')
old_issue = IssueFactory(title=u'Old Issue', project_id=1)
- self.db.session.flush()
+ self.db.session.add(old_organization)
+ self.db.session.add(old_project)
+ self.db.session.add(old_event)
+ self.db.session.add(old_issue)
+ self.db.session.commit()
+
+ from app import Organization, Project, Event, Issue
+ # make sure old org is there
+ filter = Organization.name == u'Old Organization'
+ organization = self.db.session.query(Organization).filter(filter).first()
+ self.assertIsNotNone(organization)
+
+ # make sure old project is there
+ filter = Project.name == u'Old Project'
+ project = self.db.session.query(Project).filter(filter).first()
+ self.assertIsNotNone(project)
+
+ # make sure the old issue is there
+ filter = Issue.title == u'Old Issue'
+ issue = self.db.session.query(Issue).filter(filter).first()
+ self.assertIsNotNone(issue)
+
+ # make sure old event is there
+ filter = Event.name == u'Old Event'
+ event = self.db.session.query(Event).filter(filter).first()
+ self.assertIsNotNone(event)
+
+ #
+ # run update
with HTTMock(self.response_content):
import run_update
run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
- self.db.session.flush()
-
- from app import Organization, Project, Event, Issue
-
# make sure old org is no longer there
filter = Organization.name == u'Old Organization'
organization = self.db.session.query(Organization).filter(filter).first()
@@ -294,17 +338,18 @@ def test_main_with_good_new_data(self):
event = self.db.session.query(Event).filter(filter).first()
self.assertIsNone(event)
- # check for the one organization
+ #
+ # check for one organization
filter = Organization.name == u'Cöde for Ameriça'
organization = self.db.session.query(Organization).filter(filter).first()
self.assertEqual(organization.name, u'Cöde for Ameriça')
- # check for the one project
+ # check for one project
filter = Project.name == u'bizfriendly-web'
project = self.db.session.query(Project).filter(filter).first()
self.assertEqual(project.name, u'bizfriendly-web')
- # check for the one issue
+ # check for one issue
filter = Issue.title == u'Important cityvoice issue'
issue = self.db.session.query(Issue).filter(filter).first()
self.assertEqual(issue.title, u'Important cityvoice issue')
@@ -319,7 +364,8 @@ def test_main_with_good_new_data(self):
# Thu, 16 Jan 2014 19:00:00 -05:00
self.assertEqual(first_event.utc_offset, -5 * 3600)
self.assertEqual(first_event.start_time_notz, datetime.datetime(2014, 1, 16, 19, 0, 0))
- self.assertEqual(first_event.name,u'Organizational meeting')
+ self.assertEqual(first_event.end_time_notz, datetime.datetime(2014, 1, 16, 22, 0, 0))
+ self.assertEqual(first_event.name, u'Organizational meeting')
second_event = events.pop(0)
# Thu, 20 Feb 2014 18:30:00 -05:00
@@ -367,7 +413,7 @@ def overwrite_response_content(url, request):
with HTTMock(self.response_content):
with HTTMock(overwrite_response_content):
import run_update
- self.assertFalse(run_update.github_throttling)
+ self.assertFalse(run_update.GITHUB_THROTTLING)
with self.assertRaises(IOError):
run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
@@ -377,8 +423,8 @@ def test_main_with_weird_organization_name(self):
self.setup_mock_rss_response()
def overwrite_response_content(url, request):
- if "docs.google.com" in url:
- return response(200, '''name\nCode_for-America''')
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, '''[{"name": "Code_for-America"}]''', {'content-type': 'text/csv; charset=UTF-8'})
with HTTMock(self.response_content):
with HTTMock(overwrite_response_content):
@@ -402,7 +448,7 @@ def test_main_with_bad_organization_name(self):
self.setup_mock_rss_response()
def overwrite_response_content(url, request):
- return response(200, '''name\nCode#America\nCode?America\nCode/America\nCode for America''')
+ return response(200, '''[{"name": "Code#America"}, {"name": "Code?America"}, {"name": "Code/America"}, {"name": "Code for America"}]''', {'content-type': 'text/csv; charset=UTF-8'})
with HTTMock(self.response_content):
with HTTMock(overwrite_response_content):
@@ -425,8 +471,8 @@ def test_main_with_bad_events_url(self):
self.setup_mock_rss_response()
def overwrite_response_content(url, request):
- if "docs.google.com" in url:
- return response(200, '''name,events_url\nCode for America,http://www.meetup.com/events/foo-%%%''')
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, '''[{"name": "Code for America", "events_url": "http://www.meetup.com/events/foo-%%%"}]''', {'content-type': 'text/csv; charset=UTF-8'})
logging.error = Mock()
@@ -450,9 +496,8 @@ def test_main_with_non_existant_meetup(self):
self.setup_mock_rss_response()
def overwrite_response_content(url, request):
- if "docs.google.com" in url:
- return response(200, '''name,events_url\nCode for America,http://www.meetup.com/events/Code-For-Charlotte''')
-
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, '''[{"name": "Code for America", "events_url": "http://www.meetup.com/events/Code-For-Charlotte"}]''', {'content-type': 'text/csv; charset=UTF-8'})
if 'api.meetup.com' in url:
return response(404, '''Not Found!''')
@@ -497,7 +542,7 @@ def test_github_throttling(self):
def overwrite_response_content(url, request):
if url.netloc == 'api.github.com':
- return response(403, "", {"x-ratelimit-remaining": 0})
+ return response(403, "", {"X-Ratelimit-Remaining": '0'})
with HTTMock(self.response_content):
with HTTMock(overwrite_response_content):
@@ -513,6 +558,29 @@ def overwrite_response_content(url, request):
error = self.db.session.query(Error).first()
self.assertEqual(error.error, "IOError: We done got throttled by GitHub")
+ def test_unthrottled_forbidden(self):
+ ''' A 403 response that's not due to GitHub throttling doesn't generate an error.
+ '''
+ self.setup_mock_rss_response()
+
+ def overwrite_response_content(url, request):
+ if url.netloc == 'api.github.com':
+ return response(403, "", {"X-Ratelimit-Remaining": '3388'})
+
+ with HTTMock(self.response_content):
+ with HTTMock(overwrite_response_content):
+ import run_update
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ from app import Project
+ projects = self.db.session.query(Project).all()
+ for project in projects:
+ self.assertIsNone(project.github_details)
+
+ from app import Error
+ error = self.db.session.query(Error).first()
+ self.assertIsNone(error)
+
def test_csv_sniffer(self):
'''
Testing weird csv dialects we've encountered
@@ -595,6 +663,79 @@ def updated_status(url, request):
self.assertEqual(projects[0]['status'], "active")
self.assertEqual(projects[0]['last_updated'], datetime.datetime.now().strftime("%a, %d %b %Y %H:%M:%S %Z"))
+ def test_non_github_projects_same_name(self):
+ ''' Test that non github projects with same name but different groups dont overlap
+ '''
+ self.setup_mock_rss_response()
+
+ from test.factories import OrganizationFactory
+ philly = OrganizationFactory(name=u'Code for Philly', projects_list_url=u'http://codeforphilly.org/projects.csv')
+ philly2 = OrganizationFactory(name=u'Philly2', projects_list_url=u'http://codeforphilly.org/projects.csv')
+
+ # Get a Philly project into the db
+ with HTTMock(self.response_content):
+ import run_update
+
+ # mock the time
+ freezer = freeze_time("2012-01-14 12:00:01")
+ freezer.start()
+
+ projects = run_update.get_projects(philly)
+ for proj_info in projects:
+ run_update.save_project_info(self.db.session, proj_info)
+ self.db.session.flush()
+
+ projects = run_update.get_projects(philly2)
+ for proj_info in projects:
+ run_update.save_project_info(self.db.session, proj_info)
+ self.db.session.flush()
+
+ from app import Project
+ projects = self.db.session.query(Project).all()
+ self.assertEqual(projects[0].last_updated, datetime.datetime.now().strftime("%a, %d %b %Y %H:%M:%S %Z"))
+ self.assertEqual(projects[1].last_updated, datetime.datetime.now().strftime("%a, %d %b %Y %H:%M:%S %Z"))
+
+ freezer.stop()
+ freezer = freeze_time("2012-01-14 12:00:02")
+ freezer.start()
+
+ projects = run_update.get_projects(philly)
+ for proj_info in projects:
+ run_update.save_project_info(self.db.session, proj_info)
+ self.db.session.flush()
+
+ projects = run_update.get_projects(philly2)
+ for proj_info in projects:
+ run_update.save_project_info(self.db.session, proj_info)
+ self.db.session.flush()
+
+ projects = self.db.session.query(Project).all()
+ from datetime import timedelta
+ one_second_ago = datetime.datetime.now() - timedelta(seconds=1)
+ self.assertEqual(projects[0].last_updated, one_second_ago.strftime("%a, %d %b %Y %H:%M:%S %Z"))
+ self.assertEqual(projects[1].last_updated, one_second_ago.strftime("%a, %d %b %Y %H:%M:%S %Z"))
+
+ freezer.stop()
+
+ def test_github_latest_update_time(self):
+ import run_update
+ import dateutil.parser
+ # Test that latest date is given
+ pushed_at_time = u'2015-10-02T15:43:20Z'
+ updated_at_time = u'2015-10-02T15:43:22Z'
+ github_details = {'pushed_at': pushed_at_time, 'updated_at': updated_at_time}
+ self.assertEqual(run_update.github_latest_update_time(github_details), dateutil.parser.parse(pushed_at_time).strftime('%a, %d %b %Y %H:%M:%S %Z'))
+
+ # Test handling of missing data
+ github_details = {'updated_at': updated_at_time}
+ self.assertEqual(run_update.github_latest_update_time(github_details), dateutil.parser.parse(updated_at_time).strftime('%a, %d %b %Y %H:%M:%S %Z'))
+
+ github_details = {'pushed_at': pushed_at_time}
+ self.assertEqual(run_update.github_latest_update_time(github_details), dateutil.parser.parse(pushed_at_time).strftime('%a, %d %b %Y %H:%M:%S %Z'))
+
+ github_details = {}
+ self.assertIsNotNone(run_update.github_latest_update_time(github_details))
+
def test_utf8_noncode_projects(self):
''' Test that utf8 project descriptions match exisiting projects.
'''
@@ -603,7 +744,8 @@ def test_utf8_noncode_projects(self):
from test.factories import OrganizationFactory, ProjectFactory
philly = OrganizationFactory(name=u'Code for Philly', projects_list_url=u'http://codeforphilly.org/projects.csv')
- old_project = ProjectFactory(name=u'Philly Map of Shame', organization_name=u'Code for Philly', description=u'PHL Map of Shame is a citizen-led project to map the impact of the School Reform Commission\u2019s \u201cdoomsday budget\u201d on students and parents. We will visualize complaints filed with the Pennsylvania Department of Education.', categories=u'Education, CivicEngagement', tags=u'philly, mapping', type=None, link_url=u'http://phillymapofshame.org', code_url=None, status=u'In Progress')
+ old_project = ProjectFactory(name=u'Philly Map of Shame', organization_name=u'Code for Philly', description=u'PHL Map of Shame is a citizen-led project to map the impact of the School Reform Commission\u2019s \u201cdoomsday budget\u201d on students and parents. We will visualize complaints filed with the Pennsylvania Department of Education.', categories=u'Education, CivicEngagement', tags=[u'philly', u'mapping'], type=None, link_url=u'http://phillymapofshame.org', code_url=None, status=u'In Progress')
+ old_project.last_updated = "2000-01-01"
self.db.session.flush()
def overwrite_response_content(url, request):
@@ -615,7 +757,7 @@ def overwrite_response_content(url, request):
import run_update
projects = run_update.get_projects(philly)
# If the two descriptions are equal, it won't update last_updated
- assert projects[0]['last_updated'] == None
+ self.assertEqual(projects[0]['last_updated'], "2000-01-01")
def test_issue_paging(self):
''' test that issues are following page links '''
@@ -624,23 +766,23 @@ def test_issue_paging(self):
from test.factories import OrganizationFactory, ProjectFactory
organization = OrganizationFactory(name=u'Code for America', projects_list_url=u'http://codeforamerica.org/projects.csv')
- project = ProjectFactory(organization_name=u'Code for America',code_url=u'https://github.com/TESTORG/TESTPROJECT')
- self.db.session.flush()
+ project = ProjectFactory(organization_name=organization.name, code_url=u'https://github.com/TESTORG/TESTPROJECT')
+ self.db.session.commit()
def overwrite_response_content(url, request):
if url.geturl() == 'https://api.github.com/repos/TESTORG/TESTPROJECT/issues':
- content = '''[{"number": 2,"title": "TEST TITLE 2","body": "TEST BODY 2","labels": [], "html_url":""}]'''
+ content = '''[{"number": 2,"title": "TEST TITLE 2", "created_at":"2015-10-26T18:00:00Z", "updated_at":"2015-10-26T18:06:54Z", "body": "TEST BODY 2","labels": [], "html_url":""}]'''
headers = {"Link": '"; rel="next"', 'ETag': '8456bc53d4cf6b78779ded3408886f82'}
return response(200, content, headers)
elif url.geturl() == 'https://api.github.com/repos/TESTORG/TESTPROJECT/issues?page=2':
- content = '''[{"number": 2,"title": "TEST TITLE 2","body": "TEST BODY 2","labels": [], "html_url":""}]'''
+ content = '''[{"number": 2,"title": "TEST TITLE 2", "created_at":"2015-10-26T18:00:00Z", "updated_at":"2015-10-26T18:06:54Z","body": "TEST BODY 2","labels": [], "html_url":""}]'''
return response(200, content)
with HTTMock(self.response_content):
with HTTMock(overwrite_response_content):
import run_update
- issues = run_update.get_issues(organization.name)
+ issues = run_update.get_issues(project)
assert (len(issues) == 2)
def test_project_list_without_all_columns(self):
@@ -671,11 +813,11 @@ def test_new_value_in_csv_project_list(self):
from app import Project
import run_update
- org_csv = '''name,website,events_url,rss,projects_list_url\nOrganization Name,,,,http://organization.org/projects.csv'''
+ org_json = '''[{"name": "Organization Name", "website": "", "events_url": "", "rss": "", "projects_list_url": "http://organization.org/projects.csv"}]'''
def status_one_response_content(url, request):
- if "docs.google.com" in url.geturl():
- return response(200, org_csv, {'content-type': 'text/csv; charset=UTF-8'})
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, org_json, {'content-type': 'text/csv; charset=UTF-8'})
# return an empty civic.json so the value of status there won't overwrite the one from the spreadsheet
elif "/contents/civic.json" in url.geturl():
return response(200, '''{}''', {'Etag': '8456bc53d4cf6b78779ded3408886f82'})
@@ -702,8 +844,8 @@ def status_one_response_content(url, request):
cv_headers_dict = got.headers
def status_two_response_content(url, request):
- if "docs.google.com" in url.geturl():
- return response(200, org_csv, {'content-type': 'text/csv; charset=UTF-8'})
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, org_json, {'content-type': 'text/csv; charset=UTF-8'})
# return an empty civic.json so the value of status there won't overwrite the one from the spreadsheet
elif "/contents/civic.json" in url.geturl():
return response(200, '''{}''', {'Etag': '8456bc53d4cf6b78779ded3408886f82'})
@@ -945,6 +1087,10 @@ def check_database_against_input(self):
for event_dict in check_events[organization.name]:
event = self.db.session.query(Event).filter(Event.event_url == event_dict['event_url'], Event.organization_name == event_dict['organization_name']).first()
self.assertIsNotNone(event)
+ self.assertIsNotNone(event.location)
+ self.assertIsNotNone(event.lat)
+ self.assertIsNotNone(event.lon)
+ self.assertIsNotNone(event.description)
self.assertTrue(event.keep)
# get the matching STORIES for this organization from the database
@@ -1070,6 +1216,9 @@ def test_empty_project_values_set_null(self):
def overwrite_response_content(url, request):
if "cityvoice/contents/civic.json" in url.geturl():
return response(200, '''{"status": "", "tags": ["", "", ""]}''', {'Etag': '8456bc53d4cf6b78779ded3408886f82'})
+ if url.geturl() == 'http://example.com/cfa-projects.csv':
+ project_lines = ['''Name,description,link_url,code_url,type,categories,tags,status''', ''',,,https://github.com/codeforamerica/cityvoice,,,"safety, police, poverty",Shuttered''', ''',,,https://github.com/codeforamerica/bizfriendly-web/,,,"",''']
+ return response(200, '''\n'''.join(project_lines), {'content-type': 'text/csv; charset=UTF-8'})
with HTTMock(self.response_content):
with HTTMock(overwrite_response_content):
@@ -1084,6 +1233,7 @@ def overwrite_response_content(url, request):
self.assertEqual(project.status, None)
self.assertEqual(project.tags, None)
+
# and in the saved project you know doesn't have status & tags set because they're
# missing from civic.json
filter = [Project.organization_name == u'Code for America (3)', Project.name == u'cityvoice']
@@ -1128,10 +1278,10 @@ def test_bad_events_json(self):
self.setup_mock_rss_response()
def overwrite_response_content(url, request):
- if 'meetup.com' in url.geturl() and 'Code-For-Charlotte' in url.geturl():
+ if 'https://api.meetup.com/2/events?status=past,upcoming&format=json&group_urlname=' in url.geturl() and 'Code-For-Charlotte' in url.geturl():
return response(200, 'no json object can be decoded from me')
- elif 'meetup.com' in url.geturl() and 'Code-For-Rhode-Island' in url.geturl():
+ elif 'https://api.meetup.com/2/events?status=past,upcoming&format=json&group_urlname=' in url.geturl() and 'Code-For-Rhode-Island' in url.geturl():
return response(200, None)
with HTTMock(self.response_content):
@@ -1143,6 +1293,80 @@ def overwrite_response_content(url, request):
from app import Event
self.assertEqual(self.db.session.query(Event).count(), 0)
+ def test_secondary_github_urls_handled_correctly(self):
+ ''' Projects with secondary GitHub URLs as their main URL are handled correctly.
+ '''
+ self.setup_mock_rss_response()
+
+ from app import Project
+ import run_update
+
+ # alter responses to return only one organization, with one project that
+ # has a 2nd-level GitHub URL (with /issues at the end)
+ def overwrite_response_content(url, request):
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, '''[{"name": "Cöde for Ameriça", "website": "http://codeforamerica.org", "events_url": "http://www.meetup.com/events/Code-For-Charlotte/", "rss": "http://www.codeforamerica.org/blog/feed/", "projects_list_url": "http://example.com/cfa-projects.csv"}]''', {'content-type': 'text/csv; charset=UTF-8'})
+ elif url.geturl() == 'http://example.com/cfa-projects.csv':
+ project_lines = ['''Name,description,link_url,code_url,type,categories,tags,status'''.encode('utf8'), ''',,,https://github.com/codeforamerica/cityvoice/issues,,,"safety, police, poverty",Shuttered'''.encode('utf8')]
+ return response(200, '''\n'''.join(project_lines), {'content-type': 'text/csv; charset=UTF-8'})
+
+ # run a standard run_update
+ with HTTMock(self.response_content):
+ with HTTMock(overwrite_response_content):
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ check_project = self.db.session.query(Project).first()
+ # the project exists
+ self.assertIsNotNone(check_project)
+ self.assertIsNotNone(check_project.id)
+ # the project has issues
+ self.assertTrue(hasattr(check_project, 'issues'))
+ self.assertTrue(len(check_project.issues) > 0)
+ # the project has status & tags from civic.json
+ self.assertTrue(check_project.status is not None)
+ self.assertTrue(type(check_project.status) is unicode)
+ self.assertTrue(len(check_project.status) > 0)
+ self.assertTrue(check_project.tags is not None)
+ self.assertTrue(type(check_project.tags) is list)
+ self.assertTrue(len(check_project.tags) > 0)
+
+ def test_git_extension_stripped_from_git_url(self):
+ ''' A .git extension is stripped from a project's GitHub URL
+ '''
+ self.setup_mock_rss_response()
+
+ from app import Project
+ import run_update
+
+ # alter responses to return only one organization, with one project that
+ # has a GitHub URL with .git at the end
+ def overwrite_response_content(url, request):
+ if url.geturl() == 'https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json':
+ return response(200, '''[{"name": "Cöde for Ameriça", "website": "http://codeforamerica.org", "events_url": "http://www.meetup.com/events/Code-For-Charlotte/", "rss": "http://www.codeforamerica.org/blog/feed/", "projects_list_url": "http://example.com/cfa-projects.csv"}]''', {'content-type': 'text/csv; charset=UTF-8'})
+ elif url.geturl() == 'http://example.com/cfa-projects.csv':
+ project_lines = ['''Name,description,link_url,code_url,type,categories,tags,status'''.encode('utf8'), ''',,,https://github.com/codeforamerica/cityvoice.git,,,"safety, police, poverty",Shuttered'''.encode('utf8')]
+ return response(200, '''\n'''.join(project_lines), {'content-type': 'text/csv; charset=UTF-8'})
+
+ # run a standard run_update
+ with HTTMock(self.response_content):
+ with HTTMock(overwrite_response_content):
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ check_project = self.db.session.query(Project).first()
+ # the project exists
+ self.assertIsNotNone(check_project)
+ self.assertIsNotNone(check_project.id)
+ # the project has issues
+ self.assertTrue(hasattr(check_project, 'issues'))
+ self.assertTrue(len(check_project.issues) > 0)
+ # the project has status & tags from civic.json
+ self.assertTrue(check_project.status is not None)
+ self.assertTrue(type(check_project.status) is unicode)
+ self.assertTrue(len(check_project.status) > 0)
+ self.assertTrue(check_project.tags is not None)
+ self.assertTrue(type(check_project.tags) is list)
+ self.assertTrue(len(check_project.tags) > 0)
+
def test_unmodified_projects_stay_in_database(self):
''' Verify that unmodified projects are not deleted from the database
'''
@@ -1205,69 +1429,7 @@ def test_values_set_from_civic_json(self):
project = self.db.session.query(Project).first()
self.assertIsNotNone(project)
self.assertEqual(project.status, u'Beta')
- self.assertEqual(project.tags, u'mapping,transportation,community organizing')
-
- def test_new_values_in_civic_json(self):
- ''' A value that has changed in civic.json should be saved, even if the
- related GitHub project reports that it hasn't been updated
- '''
- self.setup_mock_rss_response()
-
- from app import Project
- import run_update
-
- org_csv = '''name,website,events_url,rss,projects_list_url\nOrganization Name,,,,http://example.com/cfa-projects.csv'''
-
- # set results_state to 'after' so we'll only get one project
- self.results_state = 'after'
-
- def status_one_response_content(url, request):
- if "docs.google.com" in url.geturl():
- return response(200, org_csv, {'content-type': 'text/csv; charset=UTF-8'})
-
- with HTTMock(self.response_content):
- with HTTMock(status_one_response_content):
- run_update.main(org_name=u"Organization Name", org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
-
- project_v1 = self.db.session.query(Project).first()
- # the project status was correctly set
- self.assertEqual(project_v1.status, u'Beta')
- # the project tags were correctly set
- self.assertEqual(project_v1.tags, u'mapping,transportation,community organizing')
- v1_github_details = project_v1.github_details
-
- # save the default github response so we can send it with a 304 status below
- cv_body_text = None
- cv_headers_dict = None
- with HTTMock(self.response_content):
- from requests import get
- got = get('https://api.github.com/repos/codeforamerica/cityvoice')
- cv_body_text = str(got.text)
- cv_headers_dict = got.headers
-
- def status_two_response_content(url, request):
- if "docs.google.com" in url.geturl():
- return response(200, org_csv, {'content-type': 'text/csv; charset=UTF-8'})
- # return a civic.json with a new status value
- elif "/contents/civic.json" in url.geturl():
- return response(200, '''{"status": "Cromulent", "tags": ["community organizing", "safety and justice"]}''', {'Etag': '8456bc53d4cf6b78779ded3408886f82'})
- # return a 304 (not modified) instead of a 200 for the project
- elif url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice':
- return response(304, cv_body_text, cv_headers_dict)
-
- with HTTMock(self.response_content):
- with HTTMock(status_two_response_content):
- run_update.main(org_name=u"Organization Name", org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
-
- project_v2 = self.db.session.query(Project).first()
- # the new project status was correctly set
- self.assertEqual(project_v2.status, u'Cromulent')
- # the new tags were correctly set
- self.assertEqual(project_v2.tags, u'community organizing,safety and justice')
- # the untouched details from the GitHub project weren't changed
- self.assertEqual(project_v2.github_details, v1_github_details)
-
- self.results_state = 'before'
+ self.assertEqual(project.tags, [u'mapping', u'transportation', u'community organizing'])
def test_unicode_values_in_civic_json(self):
''' Unicode values in the civic.json file are handled correctly
@@ -1290,10 +1452,10 @@ def unicode_response_content(url, request):
project = self.db.session.query(Project).first()
self.assertIsNotNone(project)
self.assertEqual(project.status, u'汉语 漢語')
- self.assertEqual(project.tags, u'한국어 조선말,ру́сский язы́к,†≈ç®åz¥≈†')
+ self.assertEqual(project.tags, [u'한국어 조선말', u'ру́сский язы́к', u'†≈ç®åz¥≈†'])
# testing for the roman text representations as well, just for reference
self.assertEqual(project.status, u'\u6c49\u8bed \u6f22\u8a9e')
- self.assertEqual(project.tags, u'\ud55c\uad6d\uc5b4 \uc870\uc120\ub9d0,\u0440\u0443\u0301\u0441\u0441\u043a\u0438\u0439 \u044f\u0437\u044b\u0301\u043a,\u2020\u2248\xe7\xae\xe5z\xa5\u2248\u2020')
+ self.assertEqual(project.tags, [u'\ud55c\uad6d\uc5b4 \uc870\uc120\ub9d0', u'\u0440\u0443\u0301\u0441\u0441\u043a\u0438\u0439 \u044f\u0437\u044b\u0301\u043a', u'\u2020\u2248\xe7\xae\xe5z\xa5\u2248\u2020'])
def test_alt_tag_format_in_civic_json(self):
''' Tags represented as objects rather than strings are read correctly.
@@ -1312,17 +1474,17 @@ def unicode_response_content(url, request):
with HTTMock(unicode_response_content):
run_update.main(org_name=u"C\xf6de for Ameri\xe7a", org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
# check a project for the status and tags from the mock civic.json
project = self.db.session.query(Project).first()
self.assertIsNotNone(project)
self.assertEqual(project.status, u'Cromulent')
- self.assertEqual(project.tags, u'economic development,twitter,người máy,python')
+ self.assertEqual(project.tags, [u'economic development',u'twitter',u'người máy',u'python'])
# testing for the roman text representations as well, just for reference
- self.assertEqual(project.tags, u'economic development,twitter,ng\u01b0\u1eddi m\xe1y,python')
+ self.assertEqual(project.tags, [u'economic development',u'twitter',u'ng\u01b0\u1eddi m\xe1y',u'python'])
- def test_civic_json_values_preferred(self):
- ''' Values set in civic.json are preferred over values set in spreadsheets,
- even after multiple updates.
+ def test_spreadsheet_values_preferred(self):
+ ''' Values set in spreadsheet are preferred over values set in civic.json
'''
self.setup_mock_rss_response()
@@ -1340,7 +1502,7 @@ def test_civic_json_values_preferred(self):
project = self.db.session.query(Project).first()
self.assertIsNotNone(project)
self.assertEqual(project.status, u'Beta')
- self.assertEqual(project.tags, u'mapping,transportation,community organizing')
+ self.assertEqual(project.tags, [u'mapping',u'transportation',u'community organizing'])
# respond to requests for project, root file listing, and civic.json with 304s
# only if a 'If-None-Match' or 'If-Modified-Since' header is passed
@@ -1363,61 +1525,151 @@ def files_not_updated(url, request):
# check a project for the status and tags from the mock civic.json
project = self.db.session.query(Project).first()
self.assertIsNotNone(project)
- self.assertEqual(project.status, u'Beta')
- self.assertEqual(project.tags, u'mapping,transportation,community organizing')
+ self.assertEqual(project.status, u'Shuttered')
+ self.assertEqual(project.tags, [u'safety', u'police', u'poverty'])
self.results_state = 'before'
- def test_attendance(self):
- ''' Test gathering attendance from the peopledb '''
- # Mock attendance data
- cfsf_url = "https://www.codeforamerica.org/api/organizations/Code-for-San-Francisco"
- cfsf_name = "Code for San Francisco"
- oakland_url = "https://www.codeforamerica.org/api/organizations/Open-Oakland"
- oakland_name = "Open Oakland"
- cfsf_checkin1 = datetime.datetime.strptime("2015-01-01","%Y-%m-%d")
- cfsf_checkin2 = datetime.datetime.strptime("2015-01-08","%Y-%m-%d")
- oakland_checkin1 = datetime.datetime.strptime("2015-01-16","%Y-%m-%d")
- oakland_checkin2 = datetime.datetime.strptime("2015-01-24","%Y-%m-%d")
-
- # Access the peopledb
- PEOPLEDB = 'postgres:///peopledbtest'
-
- with connect(PEOPLEDB) as conn:
- with conn.cursor() as db:
- # Put some fake attendance data in it
- q = '''INSERT INTO attendance
- ( datetime, organization_url)
- VALUES ( %s, %s )'''
- db.execute(q, (cfsf_checkin1, cfsf_url))
- db.execute(q, (cfsf_checkin2, cfsf_url))
- db.execute(q, (oakland_checkin1, oakland_url))
- db.execute(q, (oakland_checkin2, oakland_url))
-
- # Call a function to pull data out of it
- with connect(PEOPLEDB) as conn:
- with conn.cursor(cursor_factory=extras.RealDictCursor) as peopledb:
+ def test_meetup_count(self):
+ ''' Test getting membership count from Meetup
+ '''
+ from test.factories import OrganizationFactory
+ org = OrganizationFactory(name="TEST ORG")
+ with HTTMock(self.response_content):
+ import run_update
+ org.member_count = run_update.get_meetup_count(organization=org, identifier="TEST-MEETUP")
+
+ self.assertEqual(org.member_count, 100)
+
+
+ def test_meetup_count_with_empty_response(self):
+ from test.factories import OrganizationFactory
+ org = OrganizationFactory(name="TEST ORG")
+ response = {
+ "status_code": 200,
+ "content": "application/json;charset=utf-8"
+ }
+ with HTTMock(lambda _url, _request: response):
+ import run_update
+ org.member_count = run_update.get_meetup_count(organization=org, identifier="TEST-MEETUP")
+
+ self.assertEqual(org.member_count, None)
+
+
+ def test_languages(self):
+ ''' Test pulling languages from Github '''
+ from app import Project
+
+ # Test that languages are returned as list
+ with HTTMock(self.response_content):
+ import run_update
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+ project = self.db.session.query(Project).first()
+ self.assertEqual(["Shell", "HTML", "Ruby", "JavaScript", "CSS"], project.languages)
+
+ # Test that null languages are handled
+ with HTTMock(self.response_content):
+
+ def overwrite_response(url, request):
+ # mock of programming languages
+ if url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice/languages':
+ return response(200, ''' { } ''')
+
+ with HTTMock(overwrite_response):
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+ project = self.db.session.query(Project).first()
+ self.assertTrue(isinstance(project.languages, type(None)))
+
+ def test_two_issues_with_the_same_name(self):
+ ''' Two issues with the same name but different html_urls should be saved as separate issues.
+ '''
+ from app import Project, Issue
+ import run_update
+ self.setup_mock_rss_response()
+
+ same_title = u'Same-Titled Cityvoice Issue'
+
+ def overwrite_response_content(url, request):
+ response_etag = {'ETag': '8456bc53d4cf6b78779ded3408886f82'}
+ if url.geturl() == 'https://api.github.com/repos/codeforamerica/cityvoice/issues':
+ return response(200, '''[{{"html_url": "https://github.com/codeforamerica/cityvoice/issue/210","title": "{issue_title}", "labels": [],"created_at": "2015-09-16T05:45:20Z", "updated_at": "2015-10-22T17:26:02Z", "body" : "WHATEVER"}}, {{"html_url": "https://github.com/codeforamerica/cityvoice/issue/211","title": "{issue_title}", "labels": [], "created_at" : "2015-10-26T01:13:03Z", "updated_at" : "2015-10-26T18:06:54Z", "body" : "WHATEVER"}}]'''.format(issue_title=same_title), response_etag)
+
+ # run a standard run_update
+ with HTTMock(self.response_content):
+ with HTTMock(overwrite_response_content):
+ run_update.main(org_name=u"Cöde for Ameriça", org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ # check the cityvoice project
+ filter = Project.name == u'cityvoice'
+ project = self.db.session.query(Project).filter(filter).first()
+ self.assertIsNotNone(project)
+ self.assertEqual(project.name, u'cityvoice')
+ project_id = project.id
+
+ # and check the issues
+ filter = Issue.title == same_title
+ issues = self.db.session.query(Issue).filter(filter).all()
+ self.assertIsNotNone(issues)
+ self.assertEqual(2, len(issues))
+ self.assertNotEqual(issues[0].html_url, issues[1].html_url)
+ for check_issue in issues:
+ self.assertEqual(check_issue.title, same_title)
+ self.assertEqual(check_issue.project_id, project_id)
+
+ def test_404ing_project_deleted(self):
+ ''' A project that once existed but is now returning a 404 is deleted from the database.
+ '''
+ from app import Project
+ self.setup_mock_rss_response()
+
+ # run a vanilla update
+ with HTTMock(self.response_content):
+ import run_update
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ filter = Project.name == u'cityvoice'
+ projects = self.db.session.query(Project).filter(filter).all()
+ self.assertEqual(len(projects), 3)
+
+ def overwrite_response_content(url, request):
+ if 'https://api.github.com/repos/codeforamerica/cityvoice' in url.geturl():
+ return response(404, '''{"message": "Not Found", "documentation_url": "https://developer.github.com/v3"}''', {'ETag': '8456bc53d4cf6b78779ded3408886f82'})
+
+ logging.error = Mock()
+
+ # run a new update
+ with HTTMock(self.response_content):
+ with HTTMock(overwrite_response_content):
import run_update
- from app import Attendance, Organization
- from test.factories import OrganizationFactory
- cfsf = OrganizationFactory(name='Code for San Francisco')
- oakland = OrganizationFactory(name='Open Oakland')
-
- cfsf_attendance = run_update.get_attendance(peopledb, cfsf_url, cfsf.name)
- self.assertEqual(cfsf_attendance["organization_name"], "Code for San Francisco")
- self.assertTrue("2015 01" in cfsf_attendance["weekly"].keys())
-
- oakland_attendance = run_update.get_attendance(peopledb, oakland_url, oakland.name)
- self.assertEqual(oakland_attendance["organization_name"], "Open Oakland")
- self.assertTrue("2015 03" in oakland_attendance["weekly"].keys())
-
- run_update.update_attendance(self.db, cfsf.name, cfsf_attendance)
- run_update.update_attendance(self.db, oakland.name, oakland_attendance)
- attendance = self.db.session.query(Attendance).all()
- self.assertEqual(attendance[0].organization_name, "Code for San Francisco")
- self.assertEqual(attendance[1].organization_name, "Open Oakland")
- self.assertTrue("2015 03" in attendance[1].weekly.keys())
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ logging.error.assert_called_with('https://api.github.com/repos/codeforamerica/cityvoice doesn\'t exist.')
+ filter = Project.name == u'cityvoice'
+ projects = self.db.session.query(Project).filter(filter).all()
+ self.assertEqual(len(projects), 0)
+
+ def test_commit_status(self):
+ """ Test grabbing the last commit status """
+ with HTTMock(self.response_content):
+ import run_update
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ from app import Project
+ filter = Project.name == u'cityvoice'
+ cityvoice = self.db.session.query(Project).filter(filter).first()
+ self.assertEqual("success", cityvoice.commit_status)
+
+ def test_logo_fetching(self):
+ """ Test grabbing the organization logo """
+ with HTTMock(self.response_content):
+ import run_update
+ run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+ from app import Organization
+ filter = Organization.name == u'Code for America (2)'
+ cfa = self.db.session.query(Organization).filter(filter).first()
+ self.assertEqual("https://avatars2.githubusercontent.com/u/337792?v=4", cfa.logo_url)
if __name__ == '__main__':
diff --git a/test_org_sources.csv b/test_org_sources.csv
index 953513e..10bbef1 100644
--- a/test_org_sources.csv
+++ b/test_org_sources.csv
@@ -1 +1 @@
-https://docs.google.com/spreadsheet/pub?key=0ArHmv-6U1drqdEVkTUtZNVlYRE5ndERLLTFDb2RqQlE&output=csv
+https://raw.githubusercontent.com/codeforamerica/brigade-information/master/test/test_organizations.json
diff --git a/utils.py b/utils.py
new file mode 100644
index 0000000..65c4307
--- /dev/null
+++ b/utils.py
@@ -0,0 +1,38 @@
+from datetime import datetime
+
+
+def is_safe_name(name):
+ ''' Return True if the string is a safe name.
+ '''
+ return raw_name(safe_name(name)) == name
+
+
+def safe_name(name):
+ ''' Return URL-safe organization name with spaces replaced by dashes.
+
+ Slashes will be removed, which is incompatible with raw_name().
+ '''
+ return name.replace(' ', '-').replace('/', '-').replace('?', '-').replace('#', '-')
+
+
+def raw_name(name):
+ ''' Return raw organization name with dashes replaced by spaces.
+
+ Also replace old-style underscores with spaces.
+ '''
+ return name.replace('_', ' ').replace('-', ' ')
+
+
+def convert_datetime_to_iso_8601(dt):
+ ''' Convert the passed datetime object to ISO 8601 format
+ '''
+ if not dt or type(dt) is not datetime:
+ return None
+
+ iso_string = unicode(dt.isoformat())
+
+ # add a 'Z' (representing the UTC time zone) to the end if there's no explicit time zone set
+ if not dt.tzinfo:
+ iso_string = u'{}Z'.format(iso_string.rstrip(u'Z'))
+
+ return iso_string
Open City 
Open City
Beta NYC
@@ -122,45 +122,7 @@ We also want to add support for many more services to be included, such as event
### How to add your Brigade to the API
-##### Brigade Information
-The new site will be powered by this [Brigade Information](https://docs.google.com/spreadsheet/ccc?key=0ArHmv-6U1drqdGNCLWV5Q0d5YmllUzE5WGlUY3hhT2c&usp=sharing) Google Spreadsheet. This way you don't need yet another account for our Brigade site. Just keep your Brigade's info up to date and you're good. Email andrewh@codeforamerica.org if you want permission to add and edit groups.
-
-The columns are:
-* Name
-* Website
-* Events Url - Point us to where ever you schedule your events. Only Meetup.com events are working right now.
-* RSS - If you have a blog, point us to it. It's pretty smart and can find the feed on its own. To show off your Google Group discussions, use a link like `https://groups.google.com/forum/feed/code-for-san-francisco/msgs/rss.xml?num=15`
-* Projects list URL - Can either be a GitHub organization url like `https://github.com/sfbrigade` or a link to a list of project URLs, described below.
-
-
-##### Projects List
-This projects list you point us to will need the following columns:
-* `name` - filled in by GitHub if left blank
-* `description` - filled in by GitHub if left blank
-* `link_url` - filled in by GitHub if left blank
-* `code_url` - Only GitHub links work for now. Others will be added as needed later.
-* `tags` - Catch-all project tagging, terms separated by commas. "Education, digital literacy, iOS, Kansas City"
-* `status` - Whatever status names you use, e.g., "Alpha, Beta, Launched, In Progress"
-
-An example:
-```
-name, description, link_url, code_url, type, categories, tags, status
-South Bend Voices, "A redeploy of CityVoice for South Bend, IN.", http://www.southbendvoices.com/, https://github.com/codeforamerica/cityvoice,,, "community engagement, housing, mapping, ruby" "In progress"
-```
-
-That projects list URL can be any flavor of csv. The easiest way is to make a Google Spreadsheet like [my example](https://docs.google.com/spreadsheet/ccc?key=0ArHmv-6U1drqdDBzNXpSZkVzRDJUQnpOS0RJM0FDWGc&usp=sharing) and then select **File > Publish it to the web**.
-
-If you are using the new Google Spreadsheets, add `/export?format=csv` to the end.
-`https://docs.google.com/spreadsheets/d/