docs: self host docs
Move doc hosting from readthedocs to espressif servers Update CI, Sphinx configs and add IDF Sphinx theme
This commit is contained in:
Marius Vikhammer
@@ -198,25 +198,17 @@ build_examples_cmake_esp32:
|
||||
|
||||
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
|
||||
|
||||
build_docs:
|
||||
.build_docs_template: &build_docs_template
|
||||
stage: build
|
||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
|
||||
image: $ESP_IDF_DOC_ENV_IMAGE
|
||||
tags:
|
||||
- build_docs
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
# English version of documentation
|
||||
- docs/en/doxygen-warning-log.txt
|
||||
- docs/en/sphinx-warning-log.txt
|
||||
- docs/en/sphinx-warning-log-sanitized.txt
|
||||
- docs/en/_build/html
|
||||
- docs/sphinx-err-*
|
||||
# Chinese version of documentation
|
||||
- docs/zh_CN/doxygen-warning-log.txt
|
||||
- docs/zh_CN/sphinx-warning-log.txt
|
||||
- docs/zh_CN/sphinx-warning-log-sanitized.txt
|
||||
- docs/zh_CN/_build/html
|
||||
- docs/*/*.txt
|
||||
- docs/_build/*/html/*
|
||||
- docs/_build/*/latex/*
|
||||
expire_in: 4 days
|
||||
only:
|
||||
variables:
|
||||
@@ -224,17 +216,31 @@ build_docs:
|
||||
- $BOT_LABEL_BUILD
|
||||
- $BOT_LABEL_BUILD_DOCS
|
||||
- $BOT_LABEL_REGULAR_TEST
|
||||
dependencies: []
|
||||
script:
|
||||
# Active python 3.6.10 env as this is where Sphinx is installed
|
||||
- source /opt/pyenv/activate && pyenv global 3.6.10
|
||||
# Setup a build dir with both languages to simplify deployment
|
||||
- cd docs
|
||||
- mkdir -p _build/$DOCLANG
|
||||
- ./check_lang_folder_sync.sh
|
||||
- cd en
|
||||
- cd $DOCLANG
|
||||
- make gh-linkcheck
|
||||
- make html
|
||||
- ../check_doc_warnings.sh
|
||||
- cd ../zh_CN
|
||||
- make gh-linkcheck
|
||||
- make html
|
||||
- make latexpdf LATEXMKOPTS="--f --interaction=nonstopmode --quiet --outdir=build"
|
||||
- ../check_doc_warnings.sh
|
||||
- mv -f _build/* ../_build/$DOCLANG
|
||||
|
||||
build_docs_en:
|
||||
extends: .build_docs_template
|
||||
variables:
|
||||
DOCLANG: "en"
|
||||
|
||||
build_docs_zh_CN:
|
||||
extends: .build_docs_template
|
||||
variables:
|
||||
DOCLANG: "zh_CN"
|
||||
|
||||
verify_cmake_style:
|
||||
extends: .check_job_template
|
||||
|
||||
@@ -73,45 +73,67 @@ push_to_github:
|
||||
- git remote add github git@github.com:espressif/esp-idf.git
|
||||
- tools/ci/push_to_github.sh
|
||||
|
||||
deploy_docs:
|
||||
.deploy_docs_template:
|
||||
extends: .before_script_lesser
|
||||
stage: deploy
|
||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
|
||||
image: $ESP_IDF_DOC_ENV_IMAGE
|
||||
tags:
|
||||
- deploy
|
||||
- shiny
|
||||
dependencies:
|
||||
- build_docs_en
|
||||
- build_docs_zh_CN
|
||||
variables:
|
||||
DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
|
||||
PYTHONUNBUFFERED: 1
|
||||
script:
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
- echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
|
||||
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
|
||||
- export GIT_VER=$(git describe --always)
|
||||
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
|
||||
|
||||
|
||||
# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
|
||||
deploy_docs_preview:
|
||||
extends: .deploy_docs_template
|
||||
only:
|
||||
refs:
|
||||
- master
|
||||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
- triggers
|
||||
variables:
|
||||
- $BOT_TRIGGER_WITH_LABEL == null
|
||||
- $BOT_LABEL_BUILD_DOCS
|
||||
dependencies:
|
||||
- build_docs
|
||||
extends: .before_script_lesser
|
||||
script:
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
- echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
|
||||
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
|
||||
- export GIT_VER=$(git describe --always)
|
||||
- cd docs/en/_build/
|
||||
- mv html $GIT_VER
|
||||
- tar czvf $GIT_VER.tar.gz $GIT_VER
|
||||
- scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
|
||||
- ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
|
||||
- cd ../../zh_CN/_build/
|
||||
- mv html $GIT_VER
|
||||
- tar czvf $GIT_VER.tar.gz $GIT_VER
|
||||
- scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
|
||||
- ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
|
||||
# add link to preview doc
|
||||
- echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
|
||||
- echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
|
||||
variables:
|
||||
TYPE: "preview"
|
||||
# older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
|
||||
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
|
||||
DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
|
||||
DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
|
||||
DOCS_DEPLOY_PATH: "$DOCS_PATH"
|
||||
DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
|
||||
|
||||
# deploy docs to production webserver
|
||||
deploy_docs_production:
|
||||
extends: .deploy_docs_template
|
||||
only:
|
||||
refs:
|
||||
# The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
|
||||
- master
|
||||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
variables:
|
||||
- $BOT_TRIGGER_WITH_LABEL == null
|
||||
variables:
|
||||
TYPE: "preview"
|
||||
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
|
||||
DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
|
||||
DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
|
||||
DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
|
||||
DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
|
||||
|
||||
deploy_test_result:
|
||||
stage: deploy
|
||||
|
||||
223
tools/ci/deploy_docs.py
Executable file
223
tools/ci/deploy_docs.py
Executable file
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# CI script to deploy docs to a webserver. Not useful outside of CI environment
|
||||
#
|
||||
#
|
||||
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import glob
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import subprocess
|
||||
import tarfile
|
||||
import packaging.version
|
||||
|
||||
|
||||
def env(variable, default=None):
|
||||
""" Shortcut to return the expanded version of an environment variable """
|
||||
return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
|
||||
|
||||
|
||||
# import sanitize_version from the docs directory, shared with here
|
||||
sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
|
||||
from sanitize_version import sanitize_version # noqa
|
||||
|
||||
|
||||
def main():
|
||||
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
|
||||
git_ver = env("GIT_VER") # output of git describe --always
|
||||
ci_ver = env("CI_COMMIT_REF_NAME", git_ver) # branch or tag we're building for (used for 'release' & URL)
|
||||
|
||||
version = sanitize_version(ci_ver)
|
||||
print("Git version: {}".format(git_ver))
|
||||
print("CI Version: {}".format(ci_ver))
|
||||
print("Deployment version: {}".format(version))
|
||||
|
||||
if not version:
|
||||
raise RuntimeError("A version is needed to deploy")
|
||||
|
||||
build_dir = env("DOCS_BUILD_DIR") # top-level local build dir, where docs have already been built
|
||||
|
||||
if not build_dir:
|
||||
raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
|
||||
|
||||
url_base = env("DOCS_DEPLOY_URL_BASE") # base for HTTP URLs, used to print the URL to the log after deploying
|
||||
|
||||
docs_server = env("DOCS_DEPLOY_SERVER") # ssh server to deploy to
|
||||
docs_user = env("DOCS_DEPLOY_SERVER_USER")
|
||||
docs_path = env("DOCS_DEPLOY_PATH") # filesystem path on DOCS_SERVER
|
||||
|
||||
if not docs_server:
|
||||
raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
|
||||
|
||||
if not docs_user:
|
||||
raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
|
||||
|
||||
docs_server = "{}@{}".format(docs_user, docs_server)
|
||||
|
||||
if not docs_path:
|
||||
raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
|
||||
|
||||
print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
|
||||
|
||||
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
|
||||
|
||||
deploy(version, tarball_path, docs_path, docs_server)
|
||||
|
||||
print("Docs URLs:")
|
||||
doc_deploy_type = os.getenv('TYPE')
|
||||
for vurl in version_urls:
|
||||
language, _, = vurl.split('/')
|
||||
tag = '{}'.format(language)
|
||||
url = "{}/{}/index.html".format(url_base, vurl) # (index.html needed for the preview server)
|
||||
url = re.sub(r"([^:])//", r"\1/", url) # get rid of any // that isn't in the https:// part
|
||||
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
|
||||
|
||||
# note: it would be neater to use symlinks for stable, but because of the directory order
|
||||
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
|
||||
# process but call the version 'stable' this time
|
||||
if is_stable_version(version):
|
||||
print("Deploying again as stable version...")
|
||||
tarball_path, version_urls = build_doc_tarball("stable", git_ver, build_dir)
|
||||
deploy("stable", tarball_path, docs_path, docs_server)
|
||||
|
||||
|
||||
def deploy(version, tarball_path, docs_path, docs_server):
|
||||
def run_ssh(commands):
|
||||
""" Log into docs_server and run a sequence of commands using ssh """
|
||||
print("Running ssh: {}".format(commands))
|
||||
subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
|
||||
|
||||
# copy the version tarball to the server
|
||||
run_ssh(["mkdir -p {}".format(docs_path)])
|
||||
print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
|
||||
subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
|
||||
|
||||
tarball_name = os.path.basename(tarball_path)
|
||||
|
||||
run_ssh(["cd {}".format(docs_path),
|
||||
"rm -rf ./*/{}".format(version), # remove any pre-existing docs matching this version
|
||||
"tar -zxvf {}".format(tarball_name), # untar the archive with the new docs
|
||||
"rm {}".format(tarball_name)])
|
||||
|
||||
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
|
||||
# as the version will be invalid for a window of time. Better to do it atomically, but this is
|
||||
# another thing made much more complex by the directory structure putting language before version...
|
||||
|
||||
|
||||
def build_doc_tarball(version, git_ver, build_dir):
|
||||
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
|
||||
the given version """
|
||||
version_paths = []
|
||||
tarball_path = "{}/{}.tar.gz".format(build_dir, version)
|
||||
|
||||
# find all the 'html/' directories under build_dir
|
||||
html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
|
||||
print("Found %d html directories" % len(html_dirs))
|
||||
|
||||
pdfs = glob.glob("{}/**/latex/build/*.pdf".format(build_dir), recursive=True)
|
||||
print("Found %d PDFs in latex directories" % len(pdfs))
|
||||
|
||||
# add symlink for stable and latest and adds them to PDF blob
|
||||
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
|
||||
|
||||
def not_sources_dir(ti):
|
||||
""" Filter the _sources directories out of the tarballs """
|
||||
if ti.name.endswith("/_sources"):
|
||||
return None
|
||||
|
||||
ti.mode |= stat.S_IWGRP # make everything group-writeable
|
||||
return ti
|
||||
|
||||
try:
|
||||
os.remove(tarball_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with tarfile.open(tarball_path, "w:gz") as tarball:
|
||||
for html_dir in html_dirs:
|
||||
# html_dir has the form '<ignored>/<language>/html/'
|
||||
language_dirname = os.path.dirname(os.path.dirname(html_dir))
|
||||
language = os.path.basename(language_dirname)
|
||||
|
||||
# when deploying, we want the top-level directory layout 'language/version'
|
||||
archive_path = "{}/{}".format(language, version)
|
||||
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
|
||||
tarball.add(html_dir, archive_path, filter=not_sources_dir)
|
||||
version_paths.append(archive_path)
|
||||
|
||||
for pdf_path in pdfs:
|
||||
# pdf_path has the form '<ignored>/<language>/<target>/latex/build'
|
||||
latex_dirname = os.path.dirname(pdf_path)
|
||||
pdf_filename = os.path.basename(pdf_path)
|
||||
language_dirname = os.path.dirname(os.path.dirname(latex_dirname))
|
||||
language = os.path.basename(language_dirname)
|
||||
|
||||
# when deploying, we want the layout 'language/version/pdf'
|
||||
archive_path = "{}/{}/{}".format(language, version, pdf_filename)
|
||||
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
|
||||
tarball.add(pdf_path, archive_path)
|
||||
|
||||
for symlink in symlinks:
|
||||
os.unlink(symlink)
|
||||
|
||||
return (os.path.abspath(tarball_path), version_paths)
|
||||
|
||||
|
||||
def create_and_add_symlinks(version, git_ver, pdfs):
|
||||
""" Create symbolic links for PDFs for 'latest' and 'stable' releases """
|
||||
|
||||
symlinks = []
|
||||
if 'stable' in version or 'latest' in version:
|
||||
for pdf_path in pdfs:
|
||||
symlink_path = pdf_path.replace(git_ver, version)
|
||||
os.symlink(pdf_path, symlink_path)
|
||||
symlinks.append(symlink_path)
|
||||
|
||||
pdfs.extend(symlinks)
|
||||
print("Found %d PDFs in latex directories after adding symlink" % len(pdfs))
|
||||
|
||||
return symlinks
|
||||
|
||||
|
||||
def is_stable_version(version):
|
||||
""" Heuristic for whether this is the latest stable release """
|
||||
if not version.startswith("v"):
|
||||
return False # branch name
|
||||
if "-" in version:
|
||||
return False # prerelease tag
|
||||
|
||||
git_out = subprocess.check_output(["git", "tag", "-l"]).decode("utf-8")
|
||||
|
||||
versions = [v.strip() for v in git_out.split("\n")]
|
||||
versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)] # include vX.Y.Z only
|
||||
|
||||
versions = [packaging.version.parse(v) for v in versions]
|
||||
|
||||
max_version = max(versions)
|
||||
|
||||
if max_version.public != version[1:]:
|
||||
print("Stable version is v{}. This version is {}.".format(max_version.public, version))
|
||||
return False
|
||||
else:
|
||||
print("This version {} is the stable version".format(version))
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -40,6 +40,7 @@ tools/ci/check_examples_cmake_make.sh
|
||||
tools/ci/check_idf_version.sh
|
||||
tools/ci/check_ut_cmake_make.sh
|
||||
tools/ci/checkout_project_ref.py
|
||||
tools/ci/deploy_docs.py
|
||||
tools/ci/envsubst.py
|
||||
tools/ci/get-full-sources.sh
|
||||
tools/ci/get_supported_examples.sh
|
||||
|
||||
Reference in New Issue
Block a user