Class GithubSearch

app/services/githubSearch.py:149–208

class GithubSearch(object):
    def __init__(self, query):
        self.results = []
        self.query = query
        language_query = "language:Dockerfile " \
                         "language:\"Java Properties\" language:\"Protocol Buffer\" " \
                         "language:Gradle language:\"Maven POM\""

        language_query_2 = "language:Python language:\"Git Config\" " \
                           "language:INI language:Shell  language:\"SSH Config\""

        extension_query = "extension:java extension:js extension:json " \
                          "extension:sql extension:yaml extension:yml extension:conf extension:config " \
                          "extension:jsp"

        extension_query_2 = "extension:php extension:py extension:go extension:bat extension:cfg " \
                            "extension:env extension:exs extension:ini " \
                            "extension:pem extension:ppk extension:cs"

        self.built_in_rules = [language_query, language_query_2, extension_query, extension_query_2]

        self.max_page = 3  # 最大翻页数目
        self.per_page = 100  # 每页数目
        self.total_count = 0  # 保存搜索结果总数

    def search(self):
        try:
            search_cnt = 0
            total_search_cnt = len(self.built_in_rules)
            for build_in in self.built_in_rules:
                search_cnt += 1
                build_in = build_in.strip()
                if not build_in:
                    continue

                curr_page = 1
                query = "{} {}".format(self.query, build_in)
                results, total_count = github_search_code(query=query, per_page=self.per_page, page=curr_page)
                logger.info("[{}/{}] page:1 keyword:{} total:{}".format(search_cnt,
                                                                        total_search_cnt,  self.query, total_count))

                self.total_count += total_count

                # 对于根据文件来查的，应该只跑一页
                if "filename:" not in build_in:
                    while (total_count / 100) > curr_page and curr_page < self.max_page:
                        curr_page += 1
                        next_results, total_count = github_search_code(query=query, per_page=self.per_page, page=curr_page)
                        logger.info("page:{} keyword:{} total:{}".format(curr_page, self.query, total_count))
                        results.extend(next_results)

                for result in results:
                    if result not in self.results:
                        self.results.append(result)
        except Exception as e:
            logger.info("Error on {} {}".format(self.query, e))
            logger.exception(e)