[Feature] Windows support (#28)

Summary: Windows support

Test Plan: All tests on Windows, Linux and Darwin

Reviewed-by: -

Issue: closes #5

Author: oraluben

.github/workflows/check.yml

@@ -19,7 +19,7 @@ jobs:
         # for actions/setup-python:
         # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
         python: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
-        os: [ ubuntu-latest, macOS-latest ]
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v4
@@ -38,7 +38,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest, macOS-latest ]
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
         conda-py: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
 
     steps:

.github/workflows/performance.yml

@@ -12,7 +12,10 @@ concurrency:
 
 jobs:
   perf-import:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v4
@@ -28,7 +31,10 @@ jobs:
           sh scripts/perf_import.sh
 
   pyperformance:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest, macOS-latest ]
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v4

CMakeLists.txt

@@ -2,7 +2,11 @@ cmake_minimum_required(VERSION 3.20)
 project(pycds)
 
 set(CMAKE_C_STANDARD 11)
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror=implicit-function-declaration")
+if (MSVC)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /we4013")
+else ()
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror=implicit-function-declaration")
+endif ()
 
 # Only for IDE linting, skbuild will handle wheel dependencies.
 if (NOT SKBUILD)
@@ -11,16 +15,16 @@ endif ()
 
 find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development.Module)
 
-add_compile_definitions(Py_BUILD_CORE)  # for sizeof(PyGC_Head)
+add_compile_definitions(Py_BUILD_CORE Py_BUILD_CORE_MODULE)  # for sizeof(PyGC_Head)
 add_compile_definitions(NEED_OPCODE_TABLES)  # for _PyOpcode_Caches and _PyOpcode_Deopt
 
-python_add_library(_cds MODULE WITH_SOABI src/_cds/_cdsmodule.c src/_cds/lookup_table.c src/_cds/hashtable.c)
+python_add_library(_cds MODULE WITH_SOABI src/_cds/_cdsmodule.c src/_cds/lookup_table.c src/_cds/hashtable.c src/_cds/platforms.c)
 # clinic-generated files access core header.
 target_include_directories(_cds PRIVATE "${Python_INCLUDE_DIRS}/internal")
 
 # todo: if find cpython repo, run clinic
 # currently we manually run `python3 ../cpython/Tools/clinic/clinic.py --make --srcdir src/_cds`
 
-install(TARGETS _cds DESTINATION .)
+install(TARGETS _cds LIBRARY DESTINATION .)
 
 install(FILES src/artifacts/pycds.pth DESTINATION .)

noxfile.py

@@ -17,9 +17,22 @@
 RELEASE = platform.release()
 PYCDS_ROOT = os.path.dirname(__file__)
 
+GA = os.environ.get('GITHUB_ACTIONS') == 'true'
+
 CDS_PYPERFORMANCE = 'git+https://github.com/oraluben/pyperformance.git@cds'
 
 
+def _clean_nox():
+    from nox.virtualenv import shutil
+
+    shutil.rmtree(os.path.join(PYCDS_ROOT, '.nox'), ignore_errors=True)
+
+
+def ci_session_cleanup():
+    if OS == 'Windows' and GA:
+        _clean_nox()
+
+
 def _py_version(session: nox.Session):
     """
     :return: "3.9", "3.10", ...
@@ -132,9 +145,11 @@ def test_import_third_party(session: nox.Session, package):
         img = os.path.join(tmp, 'test.img')
 
         session.run('python', '-c', package.import_stmt, env={'PYCDSMODE': 'TRACE', 'PYCDSLIST': lst})
-        session.run('python', '-c', f'import cds.dump; cds.dump.run_dump("{lst}", "{img}")')
+        session.run('python', '-c', f'import cds.dump; cds.dump.run_dump({repr(lst)}, {repr(img)})')
         session.run('python', '-c', package.import_stmt, env={'PYCDSMODE': 'SHARE', 'PYCDSARCHIVE': img})
 
+        ci_session_cleanup()
+
 
     @nox.session(name=f'test_import_third_party_perf-{py}', tags=['test_import_third_party_perf'], python=py)
     @nox.parametrize('package', [package for package in PACKAGES if not package.should_skip(py)])
@@ -158,7 +173,7 @@ def test_import_third_party_perf(session: nox.Session, package):
 
         logger.info(f'start generating CDS archive for {package.name}')
         session.run('python', '-c', package.import_stmt, env={'PYCDSMODE': 'TRACE', 'PYCDSLIST': lst}, log=False)
-        session.run('python', '-c', f'import cds.dump; cds.dump.run_dump("{lst}", "{img}")', log=False)
+        session.run('python', '-c', f'import cds.dump; cds.dump.run_dump({repr(lst)}, {repr(img)})', log=False)
         session.run('python', '-c', package.import_stmt, env={'PYCDSMODE': 'SHARE', 'PYCDSARCHIVE': img}, log=False)
         logger.info(f'finish generating CDS archive for {package.name}')
 
@@ -172,6 +187,8 @@ def test_import_third_party_perf(session: nox.Session, package):
                     'python', '-c', package.import_stmt,
                     env={'PYCDSMODE': 'SHARE', 'PYCDSARCHIVE': img})
 
+        ci_session_cleanup()
+
 
 def _pyperformance(session: nox.Session, pyperformance_args=None):
     session.install(CDS_PYPERFORMANCE)

pyproject.toml

@@ -6,7 +6,7 @@ requires-python = ">=3.8"
 classifiers = [
     'Development Status :: 3 - Alpha',
     'Operating System :: MacOS :: MacOS X',
-    # 'Operating System :: Microsoft :: Windows',
+    'Operating System :: Microsoft :: Windows',
     'Operating System :: POSIX :: Linux',
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
@@ -33,5 +33,9 @@ skip = "*-musllinux*"
 [tool.cibuildwheel.macos]
 archs = ["x86_64", "universal2", "arm64"]
 
+[tool.cibuildwheel.windows]
+archs = ["AMD64", "ARM64"]
+test-skip = "*-win_arm64"
+
 [tool.scikit-build]
 wheel.packages = ["src/cds"]

src/_cds/_cdscontext.h

@@ -0,0 +1,68 @@
+#ifndef PYCDS__CDSCONTEXT_H
+#define PYCDS__CDSCONTEXT_H
+
+#include <Python.h>
+#include <stdbool.h>
+
+#include "lookup_table.h"
+#include "platforms.h"
+
+struct StringRefItem {
+    PyObject **ref;
+    struct StringRefItem *next;
+};
+
+struct StringRefList {
+    PyObject *str;
+    struct StringRefItem *refs;
+    struct StringRefList *next;
+};
+
+struct CDSArchiveHeader {
+    void *mapped_addr;
+    void *none_addr;
+    void *true_addr;
+    void *false_addr;
+    void *ellipsis_addr;
+    size_t used;
+
+    PyObject *obj;
+
+    struct StringRefList *all_string_ref;
+};
+
+struct MoveInContext {
+    int n_alloc;
+
+    table *orig_pyobject_to_in_heap_pyobject_map;
+    table *in_heap_str_to_string_ref_list_map;
+
+#if PY_MINOR_VERSION >= 12
+    PyObject *static_strings;
+#endif
+};
+
+struct CDSStatus {
+    int verbose;
+    int mode;
+
+    // Prevent re-entry of cds.init_from_env()
+    bool initialized;
+
+    long shift;
+    bool traverse_error;
+
+    const char *archive;
+    fd_type archive_fd;
+#if IS_WINDOWS
+    fd_type mapping;
+#endif
+
+    struct CDSArchiveHeader *archive_header;
+
+    PyObject *flags;
+
+    struct MoveInContext *move_in_ctx;
+};
+
+#endif  // PYCDS__CDSCONTEXT_H