scraper/code_gov/models.py
def from_github3(klass, repository, labor_hours=True):
        """
        Create CodeGovProject object from github3 Repository object
        """
        if not isinstance(repository, github3.repos.repo._Repository):
            raise TypeError('Repository must be a github3 Repository object')

        logger.info('Processing: %s', repository.full_name)

        project = klass()

        logger.debug('GitHub3: repository=%s', repository)

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.git_url
        project['description'] = repository.description

        try:
            repo_license = repository.license()
        except github3.exceptions.NotFoundError:
            logger.debug('no license found for repo=%s', repository)
            repo_license = None

        if repo_license:
            license = repo_license.license
            if license:
                logger.debug('license spdx=%s; url=%s', license.spdx_id, license.url)
                if license.url is None:
                    project['permissions']['licenses'] = [{"name": license.spdx_id}]
                else:
                    project['permissions']['licenses'] = [{"URL": license.url, "name": license.spdx_id}]
            else:
                project['permissions']['licenses'] = None

        public_server = repository.html_url.startswith('https://github.com')
        if not repository.private and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['github']
        old_accept = repository.session.headers['Accept']
        repository.session.headers['Accept'] = 'application/vnd.github.mercy-preview+json'
        topics = repository._get(repository.url + '/topics').json()
        project['tags'].extend(topics.get('names', []))
        repository.session.headers['Accept'] = old_accept

        # Hacky way to get an Organization object back with GitHub3.py >= 1.2.0
        owner_url = repository.owner.url
        owner_api_response = repository._get(owner_url)
        organization = repository._json(owner_api_response, 200)
        project['contact']['email'] = organization['email']
        project['contact']['URL'] = organization['html_url']

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = organization['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.html_url

        project['downloadURL'] = repository.downloads_url

        project['languages'] = [l for l, _ in repository.languages()]

        # project['partners'] = []

        # project['relatedCode'] = []

        # project['reusedCode'] = []

        # date: [object] A date object describing the release.
        #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
        #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
        #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
        try:
            created_at = repository.created_at.date()
        except AttributeError:
            created_at = date_parse(repository.created_at).date()
        try:
            updated_at = repository.updated_at.date()
        except AttributeError:
            updated_at = date_parse(repository.updated_at).date()

        project['date'] = {
            'created': created_at.isoformat(),
            'lastModified': updated_at.isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project
def from_github3(klass, repository, labor_hours=True):
        """
        Create CodeGovProject object from github3 Repository object
        """
        if not isinstance(repository, github3.repos.repo._Repository):
            raise TypeError('Repository must be a github3 Repository object')

        logger.info('Processing: %s', repository.full_name)

        project = klass()

        logger.debug('GitHub3: repository=%s', repository)

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.git_url
        project['description'] = repository.description

        try:
            repo_license = repository.license()
        except github3.exceptions.NotFoundError:
            logger.debug('no license found for repo=%s', repository)
            repo_license = None

        if repo_license:
            license = repo_license.license
            if license:
                logger.debug('license spdx=%s; url=%s', license.spdx_id, license.url)
                if license.url is None:
                    project['permissions']['licenses'] = [{"name": license.spdx_id}]
                else:
                    project['permissions']['licenses'] = [{"URL": license.url, "name": license.spdx_id}]
            else:
                project['permissions']['licenses'] = None

        public_server = repository.html_url.startswith('https://github.com')
        if not repository.private and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['github']
        old_accept = repository.session.headers['Accept']
        repository.session.headers['Accept'] = 'application/vnd.github.mercy-preview+json'
        topics = repository._get(repository.url + '/topics').json()
        project['tags'].extend(topics.get('names', []))
        repository.session.headers['Accept'] = old_accept

        # Hacky way to get an Organization object back with GitHub3.py >= 1.2.0
        owner_url = repository.owner.url
        owner_api_response = repository._get(owner_url)
        organization = repository._json(owner_api_response, 200)
        project['contact']['email'] = organization['email']
        project['contact']['URL'] = organization['html_url']

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = organization['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.html_url

        project['downloadURL'] = repository.downloads_url

        project['languages'] = [l for l, _ in repository.languages()]

        # project['partners'] = []

        # project['relatedCode'] = []

        # project['reusedCode'] = []

        # date: [object] A date object describing the release.
        #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
        #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
        #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
        try:
            created_at = repository.created_at.date()
        except AttributeError:
            created_at = date_parse(repository.created_at).date()
        try:
            updated_at = repository.updated_at.date()
        except AttributeError:
            updated_at = date_parse(repository.updated_at).date()

        project['date'] = {
            'created': created_at.isoformat(),
            'lastModified': updated_at.isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project
Create CodeGovProject object from github3 Repository object