generate_core_modules.py

#!/usr/bin/env python3
"""
Generate embedded core modules for moonbit-eval interpreter.
This script reads all modules from moonbitlang/core and generates
RuntimePackage definitions in the format expected by core_modules.mbt.
It honors the `options.targets` directive in each package's moon.pkg
to include only files relevant for the js target.
"""

import os
import re
from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple

# Core library path
CORE_PATH = "/Users/oboard/.moon/lib/core"
OUTPUT_FILE = "interpreter/core/core_modules.mbt"

# Target we are building for
BUILD_TARGET = "js"

def parse_targets(moon_pkg_content: str) -> Dict[str, List[str]]:
    """Parse options(targets: { ... }) from moon.pkg content.
    
    Returns a dict mapping filename -> list of target strings.
    Files not listed in targets are included for all targets.
    """
    targets = {}
    # Match options(targets: { ... }) block
    m = re.search(r'targets\s*:\s*\{([^}]*)\}', moon_pkg_content, re.DOTALL)
    if not m:
        return targets
    
    block = m.group(1)
    # Parse each entry: "filename": [ "target1", "target2" ]
    for entry_m in re.finditer(r'"([^"]+)"\s*:\s*\[([^\]]*)\]', block):
        filename = entry_m.group(1)
        targets_str = entry_m.group(2)
        target_list = [t.strip().strip('"') for t in targets_str.split(',') if t.strip().strip('"')]
        targets[filename] = target_list
    
    return targets

def should_include_file(filename: str, targets: Dict[str, List[str]], build_target: str) -> bool:
    """Determine if a file should be included for the given build target.
    
    Rules:
    - If file is not in targets dict, include for all targets.
    - If the target list starts with "not", include for targets NOT in the list.
    - Otherwise, include only for targets IN the list.
    """
    if filename not in targets:
        return True
    
    target_list = targets[filename]
    if not target_list:
        return True
    
    if target_list[0] == "not":
        # Exclude for listed targets
        return build_target not in target_list[1:]
    else:
        # Include only for listed targets
        return build_target in target_list

def read_module_info(module_dir: Path) -> Optional[Dict]:
    """Read module information from a directory."""
    pkg = "moonbitlang/core/" + module_dir.relative_to(CORE_PATH).as_posix()
    alias = module_dir.relative_to(CORE_PATH).as_posix()
    module_name = pkg.replace('/', '_')
    
    dependencies = []
    
    # Read moon.pkg to get targets
    moon_pkg_path = module_dir / "moon.pkg"
    file_targets = {}
    if moon_pkg_path.exists():
        try:
            with open(moon_pkg_path, 'r', encoding='utf-8') as f:
                pkg_content = f.read()
                file_targets = parse_targets(pkg_content)
        except (UnicodeDecodeError, FileNotFoundError):
            pass
    
    # Find all .mbt files (excluding test files)
    mbt_files = [f for f in module_dir.glob("*.mbt") 
                 if not f.name.endswith('_test.mbt') and not f.name.endswith('_wbtest.mbt')]
    
    # Collect file contents with their names, filtering by target
    file_contents = {}
    
    for mbt_file in sorted(mbt_files):
        if not should_include_file(mbt_file.name, file_targets, BUILD_TARGET):
            continue
        try:
            with open(mbt_file, 'r', encoding='utf-8') as f:
                content = f.read().strip()
                if content:
                    file_contents[mbt_file.name] = content
        except (UnicodeDecodeError, FileNotFoundError):
            continue
    
    if not file_contents:
        return None
    
    concatenated_code = '\n\n'.join(file_contents.values())
    
    return {
        'alias': alias,
        'pkg': pkg,
        'name': module_name,
        'dependencies': dependencies,
        'code': concatenated_code,
        'files': file_contents
    }

def generate_module_code(module_info: Dict) -> str:
    """Generate RuntimePackage code for a single module."""
    pkg = module_info['pkg']
    module_name = module_info['name']
    dependencies = module_info['dependencies']
    files = module_info['files']
    
    if not files:
        return ""
    
    # Generate dependencies map
    dependencies_map_entries = []
    for dep in dependencies:
        if isinstance(dep, str):
            name = dep.replace('/', '_')
            dependencies_map_entries.append(f'"{dep}": {name}_module')
        elif isinstance(dep, dict):
            name = dep["path"].replace('/', '_')
            dependencies_map_entries.append(f'"{dep["alias"]}": {name}_module')
    dependencies_map_str = ", ".join(dependencies_map_entries)
    
    # Generate files map
    files_map_entries = []
    for filename, content in files.items():
        code_lines = content.split('\n')
        formatted_code_lines = []
        for line in code_lines:
            stripped_line = line.strip()
            if not stripped_line or stripped_line.startswith('//') or stripped_line.startswith('///'):
                continue
            formatted_code_lines.append(f'    #|{line}')
        
        if formatted_code_lines:
            formatted_content = '\n'.join(formatted_code_lines)
            files_map_entries.append(f'  "{filename}": (\n{formatted_content}\n  )')
        else:
            files_map_entries.append(f'  "{filename}": ""')
    
    files_map_str = ',\n'.join(files_map_entries)
    
    module_code = f'''///|
let {module_name}_module : RuntimePackage = RuntimePackage::new(
  "{pkg}",
  deps={{ {dependencies_map_str} }},
  files={{
{files_map_str}
  }}
)'''
    
    return module_code

def generate_core_modules_map(modules: List[dict]) -> str:
    """Generate the core_modules map declaration."""
    entries = []
    for module in modules: 
        entries.append(f'"{module["alias"]}": {module["name"]}_module')
        entries.append(f'"{module["pkg"]}": {module["name"]}_module')
    map_content = ", ".join(entries)
    
    return f'''///|
let core_modules : Map[String, RuntimePackage] = {{ {map_content} }}'''

def main():
    """Main function to generate all core modules."""
    core_path = Path(CORE_PATH)
    
    if not core_path.exists():
        print(f"Error: Core path {CORE_PATH} does not exist")
        return
    
    # Get all module directories recursively
    module_dirs = []
    for root, dirs, _ in os.walk(core_path):
        root_path = Path(root)
        dirs[:] = [
            d
            for d in dirs
            if not d.startswith('.') and d not in {'coverage', '_build', 'target'}
        ]
        module_dirs.extend([root_path / d for d in dirs])
    
    print(f"Found {len(module_dirs)} modules to process")
    
    generated_modules = []
    module_codes = []
    
    for module_dir in sorted(module_dirs):
        print(f"Processing module: {module_dir.name}")
        
        module_info = read_module_info(module_dir)
        
        if module_info and module_info['code'].strip():
            module_code = generate_module_code(module_info)
            if module_code:
                module_codes.append(module_code)
                generated_modules.append(module_info)
                code_lines = len([line for line in module_info['code'].split('\n') if line.strip()])
                print(f"  Generated module with {code_lines} lines of code")
        else:
            print(f"  No code content found (filtered by target)")
    
    # Generate the complete file content
    header = '''///|
/// Auto-generated core modules for moonbit-eval interpreter
/// This file contains embedded RuntimePackage definitions for all moonbitlang/core modules
'''
    
    dummy_loc = '''///|
fn dummy_loc() -> @basic.Location {
  {
    start: { fname: "", lnum: 0, bol: 0, cnum: 0 },
    end: { fname: "", lnum: 0, bol: 0, cnum: 0 },
  }
}'''
    
    core_modules_map = generate_core_modules_map(generated_modules)
    
    full_content = "\n\n".join([
        header,
        core_modules_map,
        dummy_loc
    ] + module_codes)
    
    output_path = Path(OUTPUT_FILE)
    with open(output_path, 'w', encoding='utf-8') as f:
        f.write(full_content)
    
    module_names = [module['name'] for module in generated_modules]
    print(f"\nGenerated {len(module_names)} modules:")
    for module_name in module_names:
        print(f"  - {module_name}")
    print(f"\nOutput written to: {OUTPUT_FILE}")

if __name__ == "__main__":
    main()