import json import os import sqlite3 from sqlite3 import Error from celilo.common import saru_home_ci version_configs = [ # {"branch_ref": "maintenance-4.4", "task": "assembleCivOdkRelease", "version": "4.4"}, # {"branch_ref": "maintenance-4.5", "task": "assembleCivOdkRelease", "version": "4.5"}, # {"branch_ref": "maintenance-4.6", "task": "assembleCivOdkRelease", "version": "4.6"}, # {"branch_ref": "maintenance-4.7", "task": "assembleCivOdkRelease", "version": "4.7"}, # {"branch_ref": "maintenance-4.8", "task": "assembleCivOdkRelease", "version": "4.8.1"}, # {"branch_ref": "maintenance-4.9", "task": "assembleCivOdk", "version": "4.9"}, # {"branch_ref": "maintenance-4.10", "task": "assembleCivOdk", "version": "4.10"}, # {"branch_ref": "maintenance-5.0", "task": "assembleCivOdk", "version": "5.0"}, {"branch_ref": "maintenance-5.2-rc", "task": "assembleCivOdk", "version": "5.2"}, {"branch_ref": "maintenance-5.3-rc", "task": "assembleCivOdk", "version": "5.3"}, {"branch_ref": "maintenance-5.4-rc", "task": "assembleCivOdk", "version": "5.4"}, {"branch_ref": "maintenance-5.5-rc", "task": "assembleCivOdk", "version": "5.5"} ] """ example package used by the Somewear MDM android application """ package = { "name": "ATAK", "package": "com.atakmap.app.civ", "version": "4.9.0", "description": "ATAK Android App", "fileName": "atak-4.9.0.apk", "plugins": [ { "name": "Somewear ATAK Plugin v1.0.1", "package": "com.somewearlabs.swtak.plugin", "version": "2.0.0", "description": "Somewear ATAK Plugin 1.0.1 - Smart Backhaul SingleChat", "fileName": "somewear-atak-1.0.1.0-4.9.0-civ-debug.v2.apk" } ] } class ATAKArtifactUtils: def __init__(self): self.db_name = "atak.sqlite" self.version_table = "version" self.bucket = "somewear-public/atak" ## notion block id to append the download links to self.block_id = "88d25aeb-e87c-4f09-a2f7-581ea7f0c80f" self.secret = os.environ.get('NOTION_API_SECRET') def get_first_apk_file_name(self, dir): files = [f for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))] files = [f for f in files if f.endswith(".apk")] return files[0] def create_db_if_not_exists(self): if not os.path.exists(self.db_name): sql = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( commit_hash text PRIMARY KEY, status TEXT, version text );""" print(f"Creating sqlite db: {self.db_name}") try: conn = sqlite3.connect(self.db_name) c = conn.cursor() c.execute(sql) conn.commit() conn.close() except Error as e: print(e) def check_db(self, branch_key, commit): self.create_db_if_not_exists() return self.has_uploaded(commit) def get_metadata(self, version_dir, branch_ref, atak_version): relative_path_atak = "app/build/outputs/atak-apks/odk" relative_path_app = "app/build/outputs/apk/civ/release" atak_path = f"{version_dir}/{relative_path_atak}" relative_path_version = f"{atak_version}/{relative_path_atak}" atak_app_file_name = self.get_first_apk_file_name(f"{saru_home_ci}/{atak_path}") relative_atak_app_path = f"/{relative_path_version}/{atak_app_file_name}" file_name = f"{saru_home_ci}/{version_dir}/app/build/outputs/apk/civ/release/output-metadata.json" with open(file_name) as json_file: data = json.load(json_file) variantName = data["variantName"] app_plugin_version = data["elements"][0]["versionName"] app_file_name = data["elements"][0]["outputFile"] atak_app_file_name = self.get_first_apk_file_name(f"{saru_home_ci}/{atak_path}") relative_atak_app_path = f"/{relative_path_version}/{atak_app_file_name}" relative_plugin_path = f"/{atak_version}/{relative_path_app}/{app_file_name}" return { "name": "ATAK", "version": app_plugin_version, "package": "com.atakmap.app.civ", "fileName": atak_app_file_name, "filePath": relative_atak_app_path, "description": f"ATAK Android App", "plugins": [{ "name": f"Somewear {variantName}", "version": app_plugin_version, "versionCode": data["elements"][0]["versionCode"], "package": data["applicationId"], "variantName": data["variantName"], "fileName": app_file_name, "filePath": relative_plugin_path, "branch_key": branch_ref, "description": f"Somewear ATAK Plugin {variantName} - {app_plugin_version}" }] } def has_uploaded_current_commit(self, branch_key): os.system(f"cd {saru_home_ci} && git checkout maintenance-{branch_key}") os.system(f"cd {saru_home_ci} && git pull") commit = os.popen(f'cd {saru_home_ci} && git log --pretty=format:"%h" -n 1').read() print(f"checking commit: {commit}") return {commit, self.check_db(branch_key, commit)} def download_artifact(self, artifact_root, version_config): task = version_config["task"] branch_ref = version_config["branch_ref"] version = version_config["version"] assert task assert branch_ref assert version version_dir = f"{artifact_root}/{version}" version_dir_release = f"{artifact_root}/{version}/release" all_artifact_releases = f"{artifact_root}/releases" atak_version_artifact_out = f"{saru_home_ci}/{version_dir}/release/*.apk" os.system(f"cd {saru_home_ci} && mkdir -p {version_dir}/app/build") os.system(f"cd {saru_home_ci} && mkdir -p {all_artifact_releases}") print(f"Downloading build from task {task} branch: {branch_ref} -> {version_dir}") # app/build/outputs ensure this output directory exists on version_dir script = f"cd {saru_home_ci} && glab job artifact {branch_ref} {task} --path=\"{version_dir}\"" print(f"Running script: {script}") os.system(script) print(f"Downloading assembleProduction to {version_dir_release}") cmd = f"cd {saru_home_ci} && glab ci artifact {branch_ref} assembleCivOdk --path=\"{version_dir_release}\"" os.system(cmd) print(f"Copying to release directory {atak_version_artifact_out}") os.system( f"cd {saru_home_ci} && cp {atak_version_artifact_out} {all_artifact_releases}/") return self.get_metadata(version_dir, branch_ref, version) def save_packages(self, packages): package_manifest_location = f"{saru_home_ci}/artifacts/somewear_package_manifest.json" with open(package_manifest_location, 'w') as f: json.dump(packages, f) return package_manifest_location def has_uploaded(self, commit): conn = sqlite3.connect(self.db_name) c = conn.cursor() sql = f"SELECT * FROM {self.version_table} " f"where " f"commit_hash = '{commit} and " f"status = 'pending'" c.execute(sql) row = c.fetchone() conn.close() if row is not None: return True else: return False def upload_artifact_google_cloud_storage(self, atak_version_artifact_out): os.system(f"cd {saru_home_ci} && gcloud storage cp {atak_version_artifact_out} gs://{self.bucket}") def mark_hash_uploaded(self, commit_hash, version): conn = sqlite3.connect(self.db_name) c = conn.cursor() sql = f"INSERT INTO {self.version_table} (commit_hash, status, version) VALUES ('{commit_hash}', 'uploaded', '{version}')" c.execute(sql) def download_artifacts(self): root = f"artifacts/" os.system(f"cd {saru_home_ci} && rm -rf {root}") os.system(f"cd {saru_home_ci} && mkdir -p {root}") packages = [] for version_config in version_configs: print("Downloading artifacts for version: {}".format(version_config["version"])) meta_data = self.download_artifact(root, version_config) packages.append(meta_data) print("packages: ", packages) self.save_packages({"packages": packages}) return packages def release_internal(self, packages): self.upload_and_post_to_notion(packages) manifest_path = self.save_packages({"packages": packages}) self.upload_artifact_google_cloud_storage(manifest_path) return manifest_path # def download_and_publish(self, branch_ref): def upload_and_post_to_notion(self, packages): children = [] for package in packages: version = package["plugins"][0]["branch_key"] atak_version_artifact_out = f"{saru_home_ci}/artifacts/releases/{package['plugins'][0]['fileName']}" options = self.has_uploaded_current_commit(version) has_uploaded = list(options)[0] commit_hash = list(options)[1] if has_uploaded: print(f"Skipping {commit_hash} as it has already been uploaded") continue if os.path.exists(f"{atak_version_artifact_out}"): print(f"Artifact exists: {atak_version_artifact_out}; " f"uploading to Google Cloud Storage.") self.upload_artifact_google_cloud_storage(atak_version_artifact_out) print(f"Artifact uploaded: {atak_version_artifact_out};") self.mark_hash_uploaded(commit_hash, version) else: if has_uploaded: print(f"Artifact already uploaded: {atak_version_artifact_out};") else: print(f"Artifact does not exist: {atak_version_artifact_out}; " f"most likely the download failed.") child = { "object": "block", "type": "paragraph", "paragraph": { "rich_text": [ { "type": "text", "text": { "content": package["plugins"][0]['description'] + " " } }, { "type": "text", "text": { "content": "Download", "link": { "type": "url", "url": "https://storage.googleapis.com/somewear-public/atak/" + package["plugins"][ 0]['fileName'] } }, "annotations": { "bold": False, "italic": True, "strikethrough": False, "underline": True, "code": True, "color": "green" } } ] } } children.append(child) payload = { "after": self.block_id, "children": children } cmd = 'curl --location --request PATCH \'https://api.notion.com/v1/blocks/3d1ba77a83874a9497b89e840184c23c/children\'' \ ' --header \'Notion-Version: 2022-06-28\'' \ ' --header \'Content-Type: application/json\'' \ f' --header \'Authorization: Bearer {self.secret}\'' \ f' --data \'{json.dumps(payload)}\'' os.system(cmd) print(f"Posted to notion; stable builds https://www.notion.so/somewear/ATAK-3d1ba77a83874a9497b89e840184c23c") # json = {"branch_ref": "pro-2019-submodules", "task": "assembleCivOdk", "version": "latest"} artifact_root = "artifacts" # manifest_path = ATAKArtifactUtils().download_artifact(artifact_root, version_configs) # manifest_path = ATAKArtifactUtils().download_artifacts() # print(f"Upload finished; manifest path: {manifest_path}")