import tarfile import uuid from pathlib import Path from lightning_app.source_code import LocalSourceCodeDir def test_repository_checksum(tmp_path): """LocalRepository.checksum() generates a hash of local dir.""" repository = LocalSourceCodeDir(path=Path(tmp_path)) test_path = tmp_path / "test.txt" version_a = str(uuid.uuid4()) test_path.write_text(version_a) checksum_a = repository.version # file contents don't change; checksum is the same repository = LocalSourceCodeDir(path=Path(tmp_path)) test_path.write_text(version_a) checksum_b = repository.version assert checksum_a == checksum_b # file contents change; checksum is different repository = LocalSourceCodeDir(path=Path(tmp_path)) test_path.write_text(str(uuid.uuid4())) checksum_c = repository.version assert checksum_a != checksum_c def test_repository_package(tmp_path, monkeypatch): """LocalRepository.package() ceates package from local dir.""" cache_path = Path(tmp_path) source_path = cache_path / "nested" source_path.mkdir(parents=True, exist_ok=True) (source_path / "test.txt").write_text("test") # set cache location to temp dir monkeypatch.setattr(LocalSourceCodeDir, "cache_location", cache_path) repository = LocalSourceCodeDir(path=source_path) repository.package() # test that package is created for file in cache_path.glob("**/*"): if file.is_file() and file.name.endswith(".tar.gz"): assert file.name == f"{repository.version}.tar.gz" def test_repository_lightningignore(tmp_path): """LocalRepository.version uses the assumed checksum correctly.""" # write .lightningignore file lightningignore = """ # ignore files in this dir ignore/ """ (tmp_path / ".lightningignore").write_text(lightningignore) # write some data to file and check version (tmp_path / "test.txt").write_text(str(uuid.uuid4())) # create repo object repository = LocalSourceCodeDir(path=Path(tmp_path)) checksum_a = repository.version # write file that needs to be ignored (tmp_path / "ignore").mkdir() (tmp_path / "ignore/test.txt").write_text(str(uuid.uuid4())) # check that version remains the same repository = LocalSourceCodeDir(path=Path(tmp_path)) checksum_b = repository.version assert checksum_a == checksum_b def test_repository_filters_with_absolute_relative_path(tmp_path): """.lightningignore parsing parses paths starting with / correctly.""" lightningignore = """ /ignore_file/test.txt /ignore_dir """ (tmp_path / ".lightningignore").write_text(lightningignore) # write some data to file and check version (tmp_path / "test.txt").write_text(str(uuid.uuid4())) # create repo object repository = LocalSourceCodeDir(path=Path(tmp_path)) checksum_a = repository.version # only two files in hash assert len(repository._non_ignored_files) == 2 # write file that needs to be ignored (tmp_path / "ignore_file").mkdir() (tmp_path / "ignore_dir").mkdir() (tmp_path / "ignore_file/test.txt").write_text(str(uuid.uuid4())) (tmp_path / "ignore_dir/test.txt").write_text(str(uuid.uuid4())) # check that version remains the same repository = LocalSourceCodeDir(path=Path(tmp_path)) checksum_b = repository.version # still only two files in hash assert len(repository._non_ignored_files) == 2 assert checksum_a == checksum_b def test_repository_lightningignore_supports_different_patterns(tmp_path): """.lightningignore parsing supports different patterns.""" # write .lightningignore file # default github python .gitignore lightningignore = """ # ignore files in this dir ignore/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .env.docker .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # VS Code files .vscode/ # UI files node_modules/ # Data files models/ models/* !grid/openapi/models postgresql_data/ redis_data/ # Secrets folders secrets/ # Built UI ui/ # Ignores Grid Runner vendor/ ignore_test.py # Ignore cov report *.xml """ (tmp_path / ".lightningignore").write_text(lightningignore) # write some data to file and check version (tmp_path / "test.txt").write_text(str(uuid.uuid4())) # create repo object repository = LocalSourceCodeDir(path=Path(tmp_path)) checksum_a = repository.version # write file that needs to be ignored (tmp_path / "ignore").mkdir() (tmp_path / "ignore/test.txt").write_text(str(uuid.uuid4())) # check that version remains the same repository = LocalSourceCodeDir(path=Path(tmp_path)) checksum_b = repository.version assert checksum_a == checksum_b def test_repository_lightningignore_unpackage(tmp_path, monkeypatch): """.lightningignore behaves similarly to the gitignore standard.""" lorem_ipsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit." cache_path = tmp_path / "cache" monkeypatch.setattr(LocalSourceCodeDir, "cache_location", cache_path) source_path = tmp_path / "source" source_path.mkdir() # set cache location to temp dir lightningignore = """ # Ignore on all levels *.pyc *__pycache__/ build/ .env # Ignore wildcard on one level ./*.txt /*.md ./one-level/*.txt /one-level/*.md # Ignore only relative ./downloads /relative_downloads # nested /nested//level/ /nested/level/ """ (source_path / ".lightningignore").write_text(lightningignore) # Dir structure (source_path / "include.py").write_text(lorem_ipsum) (source_path / "exclude.pyc").write_text(lorem_ipsum) (source_path / "__pycache__").mkdir() (source_path / "__pycache__" / "exclude.py").write_text( lorem_ipsum ) # Even tho it's .py it's in excluded __pycache__ directory (source_path / "__pycache__" / "exclude.pyc").write_text( lorem_ipsum ) # Even tho it's .py it's in excluded __pycache__ directory (source_path / "build.py").write_text(lorem_ipsum) # Common prefix with excluded build but it's not it (source_path / "builds").mkdir() # Common prefix with excluded build but it's not excluded (source_path / "builds" / "include.py").write_text(lorem_ipsum) (source_path / "builds" / "__pycache__").mkdir() # Recursively excluded (source_path / "builds" / "__pycache__" / "exclude.py").write_text(lorem_ipsum) (source_path / "build").mkdir() # Recursively excluded (source_path / "build" / "exclude.db").write_text(lorem_ipsum) (source_path / ".env").write_text(lorem_ipsum) # No issues with handling hidden (.dot) files (source_path / "downloads").mkdir() # exclude (source_path / "downloads" / "something.jpeg").write_text(lorem_ipsum) (source_path / "relative_downloads").mkdir() # exclude (source_path / "relative_downloads" / "something.jpeg").write_text(lorem_ipsum) (source_path / "include").mkdir() # include (source_path / "include" / "exclude.pyc").write_text(lorem_ipsum) # exclude because of *.pyc rule (source_path / "include" / "include.py").write_text(lorem_ipsum) # include (source_path / "include" / "downloads").mkdir() # include because it was excluded only relative to root (source_path / "include" / "downloads" / "something.jpeg").write_text(lorem_ipsum) (source_path / "include" / "relative_downloads").mkdir() # include because it was excluded only relative to root (source_path / "include" / "relative_downloads" / "something.jpeg").write_text(lorem_ipsum) (source_path / "exclude.txt").write_text(lorem_ipsum) (source_path / "exclude.md").write_text(lorem_ipsum) (source_path / "one-level").mkdir() (source_path / "one-level" / "exclude.txt").write_text(lorem_ipsum) (source_path / "one-level" / "exclude.md").write_text(lorem_ipsum) (source_path / "one-level" / "include.py").write_text(lorem_ipsum) (source_path / "nested").mkdir() (source_path / "nested" / "include.py").write_text(lorem_ipsum) (source_path / "nested" / "level").mkdir() (source_path / "nested" / "level" / "exclude.py").write_text(lorem_ipsum) # create repo object repository = LocalSourceCodeDir(path=source_path) repository.package() unpackage_path = tmp_path / "unpackage" with tarfile.open(repository.package_path) as f: f.extractall(unpackage_path) assert (unpackage_path / "include.py").exists() assert not (unpackage_path / "exclude.pyc").exists() # Excluded by *.pyc assert not (unpackage_path / "__pycache__").exists() assert not ( unpackage_path / "__pycache__" / "exclude.py" ).exists() # Even tho it's .py it's in excluded __pycache__ directory assert not ( unpackage_path / "__pycache__" / "exclude.pyc" ).exists() # Even tho it's .py it's in excluded __pycache__ directory assert (unpackage_path / "build.py").exists() # Common prefix with excluded build but it's not it assert (unpackage_path / "builds" / "include.py").exists() assert not (unpackage_path / "builds" / "__pycache__").exists() # Recursively excluded assert not (unpackage_path / "builds" / "__pycache__" / "exclude.py").exists() assert not (unpackage_path / "build").exists() # Recursively excluded assert not (unpackage_path / "build" / "exclude.db").exists() assert not (unpackage_path / ".env").exists() # No issues with handling hidden (.dot) files assert not (unpackage_path / "downloads").mkdir() # exclude assert not (unpackage_path / "downloads" / "something.jpeg").exists() assert not (unpackage_path / "relative_downloads").mkdir() # exclude assert not (unpackage_path / "relative_downloads" / "something.jpeg").exists() assert not (unpackage_path / "include" / "exclude.pyc").exists() # exclude because of *.pyc rule assert (unpackage_path / "include" / "include.py").exists() # include assert ( unpackage_path / "include" / "downloads" / "something.jpeg" ).exists() # include because it was excluded only relative to root assert ( unpackage_path / "include" / "relative_downloads" / "something.jpeg" ).exists() # include because it was excluded only relative to root assert not (unpackage_path / "exclude.txt").exists() assert not (unpackage_path / "exclude.md").exists() assert not (unpackage_path / "one-level" / "exclude.txt").exists() assert not (unpackage_path / "one-level" / "exclude.md").exists() assert (unpackage_path / "one-level" / "include.py").exists() assert (unpackage_path / "nested" / "include.py").exists() assert not (unpackage_path / "nested" / "level" / "exclude.py").exists()