Gemini 1.5 One Million Sprint

import os, sys
import requests, zipfile
import io
import ast

def is_desired_file(file_path):
    """Check if the file is a Python, JavaScript, TypeScript, Svelte, or Rust file."""
    is_code = file_path.endswith(".py") or file_path.endswith(".js") or file_path.endswith(".ts") or file_path.endswith(".svelte") or file_path.endswith(".rs")
    is_docs = file_path.endswith(".md") or file_path.endswith(".mdx")
    return is_code or is_docs

def is_likely_useful_file(file_path):
    """Determine if the file is likely to be useful by excluding certain directories and specific file types."""
    excluded_dirs = ["tests", "test", "__pycache__", "node_modules", ".venv"]
    #excluded_dirs += ["docs", "examples", "benchmarks", ]
    #excluded_dirs += ["scripts", "utils", ]
    excluded_dirs += ["cache", ]

    utility_or_config_files = ["hubconf.py", "setup.py", "package-lock.json"]
    github_workflow_or_docs = ["stale.py", "gen-card-", "write_model_card"]

    if any(part.startswith('.') for part in file_path.split('/')):
        return False
    if 'test' in file_path.lower():
        return False
    for excluded_dir in excluded_dirs:
        if f"/{excluded_dir}/" in file_path or file_path.startswith(f"{excluded_dir}/"):
            return False
    for file_name in utility_or_config_files:
        if file_name in file_path:
            return False
    return all(doc_file not in file_path for doc_file in github_workflow_or_docs)

def is_test_file(file_content):
    """Determine if the file content suggests it is a test file."""
    test_indicators = ["import unittest", "import pytest", "from unittest", "from pytest"]
    return any(indicator in file_content for indicator in test_indicators)

def has_sufficient_content(file_content, min_line_count=10):
    """Check if the file has a minimum number of substantive lines."""
    lines = [line for line in file_content.split('\n') if line.strip() and not line.strip().startswith('#')]
    return len(lines) >= min_line_count

def remove_comments_and_docstrings(source):
    """Remove comments and docstrings from the Python source code."""
    tree = ast.parse(source)
    for node in ast.walk(tree):
        if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)) and ast.get_docstring(node):
            node.body = node.body[1:]  # Remove docstring
        elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Str):
            node.value.s = ""  # Remove comments
    return ast.unparse(tree)


def download_repo(repo_url, output_file):
    """Download and process files from a GitHub repository."""
    if '/tree/' in repo_url:
        repo_url = f'https://download-directory.github.io/?{repo_url}'

    response = requests.get(f"{repo_url}/archive/master.zip")
    zip_file = zipfile.ZipFile(io.BytesIO(response.content))

    with open(output_file, "w", encoding="utf-8") as outfile:
        for file_path in zip_file.namelist():
            # Skip directories, non-Python files, less likely useful files, hidden directories, and test files
            if file_path.endswith("/") or not is_desired_file(file_path) or not is_likely_useful_file(file_path):
                continue

            file_content = zip_file.read(file_path).decode("utf-8")

            # Skip test files based on content and files with insufficient substantive content
            if is_test_file(file_content) or not has_sufficient_content(file_content):
                continue

            try:
                file_content = remove_comments_and_docstrings(file_content)
            except SyntaxError:
                # Skip files with syntax errors
                continue

            outfile.write(f"# File: {file_path}\n")
            outfile.write(file_content)
            outfile.write("\n\n")



def list_dir_recursive(directory):
    files, dirs=[], []
    for entry in sorted(os.listdir(directory)):
        full_path = os.path.join(directory, entry)
        if os.path.isdir(full_path):
            dirs.append(full_path)  # Do directories after files
        else:
            files.append(full_path)
    for dir in dirs:
        files += list_dir_recursive(dir)
    return files

def extract_from_folder(folder, output_file):
    """Process files from given folder."""

    with open(output_file, "w", encoding="utf-8") as outfile:
        for file_path in list_dir_recursive(folder):
            # Skip directories, non-Python files, less likely useful files, hidden directories, and test files
            if file_path.endswith("/") or not is_desired_file(file_path) or not is_likely_useful_file(file_path):
                print(f"Reject undesirable : {file_path}")
                continue

            with open(file_path, 'rt', encoding='utf-8') as f:
                file_content = f.read()

            # Skip test files based on content and files with insufficient substantive content
            #if is_desired_file(file_content) or not has_sufficient_content(file_content):
            if is_test_file(file_content):
                print(f"Reject Test file : {file_path}")
                continue
            if not has_sufficient_content(file_content):
                print(f"MAYBE Insufficient content : {file_path}")
                #continue

            try:
                #file_content = remove_comments_and_docstrings(file_content)
                pass # Actually like the commented code...
            except SyntaxError:
                # Skip files with syntax errors
                continue

            outfile.write(f"\n# --- File: {file_path} ---\n\n")
            outfile.write(file_content)
            outfile.write("\n\n")
    print(f"Combined source code saved to {output_file}")

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python code-and-docs-from-folder.py <folder>")
        sys.exit(1)

    folder = sys.argv[1]
    output_file = f"{folder}_code.txt"
    #download_repo(repo_url, output_file)

    # git clone https://github.com/stanfordnlp/dspy.git

    # Prompt for 'hello world' DSPy code generation :
    """
    Making use of the dspy repo code shown above, 
    create a simple dspy demonstration in python 
    that allows one to input the name of a country, 
    and get back the name of the capital city 
    (this is a very simple task, using almost none of the special features, 
    but will serve as a 'hello world' example for the framework)
    """

    if False:
        print(list_dir_recursive(folder))
    if False:
        for file_path in list_dir_recursive(folder):
            if is_desired_file(file_path) and is_likely_useful_file(file_path):
                print(file_path)
    if True:
        extract_from_folder(folder, output_file)