diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 5157da536f02a853de79307a914fa8e68b76df29..6227dc256a1d856daae40634521e7fd6e26eb310 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -63,7 +63,7 @@ This change added tests and can be verified as follows:
- The public API, i.e., is any changed class annotated with `@Public(Evolving)`: (yes / no)
- The serializers: (yes / no / don't know)
- The runtime per-record code paths (performance sensitive): (yes / no / don't know)
- - Anything that affects deployment or recovery: JobManager (and its components), Checkpointing, Yarn/Mesos, ZooKeeper: (yes / no / don't know)
+ - Anything that affects deployment or recovery: JobManager (and its components), Checkpointing, Kubernetes/Yarn/Mesos, ZooKeeper: (yes / no / don't know)
- The S3 file system connector: (yes / no / don't know)
## Documentation
diff --git a/.travis.yml b/.travis.yml
index acf8b7e51aea608ede2a481aa20fdfd9e1b28ae0..8d99bda37d5178ab33c59c573a82859ed4b4ac0a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,7 @@ cache:
- $HOME/maven_cache
# keep in sync with tools/travis/docs.sh
- $HOME/gem_cache
+ - $HOME/flink_download_cache
# do not cache our own artifacts
before_cache:
@@ -41,6 +42,11 @@ git:
env:
global:
+ - cache-dir: $HOME/flink_download_cache
+ - cache-btl: 30
+ - cache-download-max-retries: 3
+ - cache-download-attempt-timeout: 5
+ - cache-download-global-timeout: 10
# Global variable to avoid hanging travis builds when downloading cache archives.
- MALLOC_ARENA_MAX=2
- DOCKER_COMPOSE_VERSION=1.22.0
@@ -112,6 +118,14 @@ jobs:
script: ./tools/travis_controller.sh tests
env: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
name: tests
+ - if: type in (pull_request, push)
+ script: ./tools/travis_controller.sh legacy_scheduler_core
+ env: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+ name: core - legacy_scheduler
+ - if: type in (pull_request, push)
+ script: ./tools/travis_controller.sh legacy_scheduler_tests
+ env: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+ name: tests - legacy_scheduler
- if: type in (pull_request, push)
script: ./tools/travis_controller.sh misc
env: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
@@ -248,7 +262,7 @@ jobs:
jdk: "openjdk11"
script: ./tools/travis_controller.sh misc
env: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11 -Djdk11"
- name: misc
+ name: misc - jdk 11
- if: type = cron
jdk: "openjdk11"
stage: cleanup
@@ -265,99 +279,115 @@ jobs:
# E2E profiles - Hadoop 2.8
- if: type = cron
stage: test
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -De2e-metrics"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis1,e2e-hadoop"
script: ./tools/travis/nightly.sh split_misc.sh
name: e2e - misc - hadoop 2.8
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis2,e2e-hadoop"
script: ./tools/travis/nightly.sh split_ha.sh
name: e2e - ha - hadoop 2.8
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis3,e2e-hadoop"
script: ./tools/travis/nightly.sh split_sticky.sh
name: e2e - sticky - hadoop 2.8
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis4,e2e-hadoop"
script: ./tools/travis/nightly.sh split_checkpoints.sh
name: e2e - checkpoints - hadoop 2.8
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis5,e2e-hadoop"
script: ./tools/travis/nightly.sh split_container.sh
name: e2e - container - hadoop 2.8
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis6,e2e-hadoop"
script: ./tools/travis/nightly.sh split_heavy.sh
name: e2e - heavy - hadoop 2.8
+ - if: type = cron
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis6,e2e-hadoop"
+ script: ./tools/travis/nightly.sh split_tpcds.sh
+ name: e2e - tpcds - hadoop 2.8
# E2E profiles - Scala 2.12
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -De2e-metrics"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis1,e2e-hadoop"
script: ./tools/travis/nightly.sh split_misc.sh
name: e2e - misc - scala 2.12
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis2,e2e-hadoop"
script: ./tools/travis/nightly.sh split_ha.sh
name: e2e - ha - scala 2.12
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis3,e2e-hadoop"
script: ./tools/travis/nightly.sh split_sticky.sh
name: e2e - sticky - scala 2.12
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis4,e2e-hadoop"
script: ./tools/travis/nightly.sh split_checkpoints.sh
name: e2e - checkpoints - scala 2.12
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis5,e2e-hadoop"
script: ./tools/travis/nightly.sh split_container.sh
name: e2e - container - scala 2.12
- if: type = cron
- env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12"
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis6,e2e-hadoop"
script: ./tools/travis/nightly.sh split_heavy.sh
name: e2e - heavy - scala 2.12
+ - if: type = cron
+ env: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dscala-2.12 -Pe2e-travis6,e2e-hadoop"
+ script: ./tools/travis/nightly.sh split_tpcds.sh
+ name: e2e - tpcds - scala 2.12
# E2E profiles - Hadoop-free
- if: type = cron
- env: PROFILE="-De2e-metrics"
+ env: PROFILE="-Pe2e-travis1"
script: ./tools/travis/nightly.sh split_misc_hadoopfree.sh
name: e2e - misc
- if: type = cron
- env: PROFILE=""
+ env: PROFILE="-Pe2e-travis2"
script: ./tools/travis/nightly.sh split_ha.sh
name: e2e - ha
- if: type = cron
- env: PROFILE=""
+ env: PROFILE="-Pe2e-travis3"
script: ./tools/travis/nightly.sh split_sticky.sh
name: e2e - sticky
- if: type = cron
- env: PROFILE=""
+ env: PROFILE="-Pe2e-travis4"
script: ./tools/travis/nightly.sh split_checkpoints.sh
name: e2e - checkpoints
- if: type = cron
- env: PROFILE=""
- script: ./tools/travis/nightly.sh split_container.sh
+ env: PROFILE="-Pe2e-travis5"
+ script: ./tools/travis/nightly.sh split_container.sh without-hadoop
name: e2e - container
- if: type = cron
- env: PROFILE=""
+ env: PROFILE="-Pe2e-travis6"
script: ./tools/travis/nightly.sh split_heavy.sh
name: e2e - heavy
+ - if: type = cron
+ env: PROFILE="-Pe2e-travis6"
+ script: ./tools/travis/nightly.sh split_tpcds.sh
+ name: e2e - tpcds
# E2E profiles - Java 11
- if: type = cron
stage: test
jdk: "openjdk11"
- env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -De2e-metrics"
+ env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis1,e2e-hadoop"
script: ./tools/travis/nightly.sh split_misc.sh
name: e2e - misc - jdk11
- if: type = cron
- env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis2,e2e-hadoop"
script: ./tools/travis/nightly.sh split_ha.sh
name: e2e - ha - jdk11
- if: type = cron
- env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis3,e2e-hadoop"
script: ./tools/travis/nightly.sh split_sticky.sh
name: e2e - sticky - jdk 11
- if: type = cron
- env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis4,e2e-hadoop"
script: ./tools/travis/nightly.sh split_checkpoints.sh
name: e2e - checkpoints - jdk 11
- if: type = cron
- env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3"
+ env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis6,e2e-hadoop"
script: ./tools/travis/nightly.sh split_heavy.sh
name: e2e - heavy - jdk 11
+ - if: type = cron
+ env: PROFILE="-Djdk11 -Dinclude-hadoop -Dhadoop.version=2.8.3 -Pe2e-travis6,e2e-hadoop"
+ script: ./tools/travis/nightly.sh split_tpcds.sh
+ name: e2e - tpcds - jdk 11
diff --git a/NOTICE b/NOTICE
index ede0dfa17caaedaf2087a13abbbf586b7a0988de..35f4f6e897016b47085aa7b1e73c60294cfa12f9 100644
--- a/NOTICE
+++ b/NOTICE
@@ -4,35 +4,9 @@ Copyright 2014-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
-This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
-
-- nvd3#1.8.4
-
-This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT)
-See bundled license files for details.
-
-- angular:1.4.8
-- angular-drag-and-drop-list:1.4.0
-- angular-moment:0.10.3
-- angular-ui-router:0.2.15
-- bootstrap:3.3.6
-- dagre:0.7.5
-- dagre-d3:0.4.17
-- ev-emitter:1.1.1
-- font-awesome:4.5.0 (CSS)
-- graphlib:1.0.7
-- imagesloaded:4.1.4
-- jquery:2.2.0
-- lodash:3.10.1
-- moment:2.10.6
-- moment-duration-format:1.3.0
-- qtip2:2.2.1
-- Split.js:1.0.6
-
This project bundles the following dependencies under the BSD license.
See bundled license files for details.
-- d3:3.5.12
- cloudpickle:1.2.2
- net.sf.py4j:py4j:0.10.8.1
@@ -40,7 +14,6 @@ This project bundles the following dependencies under SIL OFL 1.1 license (https
See bundled license files for details.
- font-awesome:4.5.0 (Font) (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
- -> fonts in "flink-runtime-web/web-dashboard/web/fonts"
-> fonts in "docs/page/font-awesome/fonts"
The Apache Flink project contains or reuses code that is licensed under the ISC license from the following projects.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index cc41f084b2df3a079d9c3d338da464713a7ee1a1..d486643a5dbf7160362a2410d07f44658f664fe3 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,23 +13,47 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#
+# This file defines an Azure Pipeline build for testing Flink. It is intended to be used
+# with a free Azure Pipelines account.
+# It has the following features:
+# - default builds for pushes / pull requests
+# - end-to-end tests
+#
+#
+# For the "apache/flink" repository, we are using the pipeline definition located in
+# tools/azure-pipelines/build-apache-repo.yml
+# That file points to custom, self-hosted build agents for faster pull request build processing and
+# integration with Flinkbot.
+# The custom pipeline definition file is configured in the "Pipeline settings" screen
+# of the Azure Pipelines web ui.
+#
-trigger:
- branches:
- include:
- - '*'
resources:
containers:
- # Container with Maven 3.2.5 to have the same environment everywhere.
+ # Container with Maven 3.2.5, SSL to have the same environment everywhere.
- container: flink-build-container
- image: rmetzger/flink-ci:3
- repositories:
- - repository: templates
- type: github
- name: flink-ci/flink-azure-builds
- endpoint: flink-ci
+ image: rmetzger/flink-ci:ubuntu-amd64-3528acd
+
+# See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
+variables:
+ MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
+ MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+ CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
+ CACHE_FALLBACK_KEY: maven | $(Agent.OS)
+ CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
-jobs:
-- template: flink-build-jobs.yml@templates
+jobs:
+ - template: tools/azure-pipelines/jobs-template.yml
+ parameters: # see template file for a definition of the parameters.
+ stage_name: ci_build
+ test_pool_definition:
+ vmImage: 'ubuntu-latest'
+ e2e_pool_definition:
+ vmImage: 'ubuntu-16.04'
+ environment: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+ run_end_to_end: false
+ container: flink-build-container
+ jdk: jdk8
diff --git a/docs/.gitignore b/docs/.gitignore
index 98b6f6b56036b0bf385fb29480e08adb82842837..3d6212de50701bdf1098749dcbbe9f8e5d5835f2 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1,6 +1,8 @@
.bundle/
.jekyll-metadata
+.jekyll-cache/
.rubydeps/
content/
+content_*/
ruby2/.bundle/
-ruby2/.rubydeps/
\ No newline at end of file
+ruby2/.rubydeps/
diff --git a/docs/404.md b/docs/404.md
index 42e390b8798ab9d319234a436b5e6a4497ea9e46..a752f8d5054a06e1c3a164b832a75032ee5edfbb 100644
--- a/docs/404.md
+++ b/docs/404.md
@@ -21,6 +21,6 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-
+
+The page you are looking for has been moved. This could be because of a recent reorganization of the
+documentation. Redirecting to [Documentation Home Page]({{ site.baseurl }}/) in 5 seconds.
diff --git a/docs/Gemfile b/docs/Gemfile
index b519eb9d578694032c88b623c7ed5735a509896a..3edc85dcd397c722a222151dd8df5315a46e4e21 100644
--- a/docs/Gemfile
+++ b/docs/Gemfile
@@ -18,17 +18,18 @@
source 'https://rubygems.org'
-ruby '>= 2.1.0'
+ruby '>= 2.4.0'
-gem 'jekyll', '3.7.2'
-gem 'addressable', '2.4.0'
-gem 'octokit', '~> 4.3.0'
-gem 'therubyracer', '0.12.2'
-gem 'json', '2.0.4'
+gem 'jekyll', '4.0.0'
+gem 'addressable', '2.7.0'
+gem 'octokit', '4.14.0'
+gem 'therubyracer', '0.12.3'
+gem 'json', '2.2.0'
gem 'jekyll-multiple-languages', '2.0.3'
gem 'jekyll-paginate', '1.1.0'
gem 'liquid-c', '4.0.0' # speed-up site generation
+gem 'sassc', '2.2.1' # speed-up site generation
-group :jekyll_plugins do
- gem 'hawkins'
-end
+# group :jekyll_plugins do
+# gem 'hawkins'
+# end
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index 09b02e8b709ac04d0caa66796fa807e148253dd0..9af2cbf85b8d61529d7de67b43461e48fb4f38b8 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -1,93 +1,94 @@
GEM
remote: https://rubygems.org/
specs:
- addressable (2.4.0)
+ addressable (2.7.0)
+ public_suffix (>= 2.0.2, < 5.0)
colorator (1.1.0)
- concurrent-ruby (1.0.5)
+ concurrent-ruby (1.1.5)
em-websocket (0.5.1)
eventmachine (>= 0.12.9)
http_parser.rb (~> 0.6.0)
- eventmachine (1.2.5)
- faraday (0.9.2)
+ eventmachine (1.2.7)
+ faraday (0.17.0)
multipart-post (>= 1.2, < 3)
- ffi (1.9.18)
+ ffi (1.11.2)
forwardable-extended (2.6.0)
- hawkins (2.0.5)
- em-websocket (~> 0.5)
- jekyll (~> 3.1)
http_parser.rb (0.6.0)
- i18n (0.9.3)
+ i18n (1.7.0)
concurrent-ruby (~> 1.0)
- jekyll (3.7.2)
+ jekyll (4.0.0)
addressable (~> 2.4)
colorator (~> 1.0)
em-websocket (~> 0.5)
- i18n (~> 0.7)
- jekyll-sass-converter (~> 1.0)
+ i18n (>= 0.9.5, < 2)
+ jekyll-sass-converter (~> 2.0)
jekyll-watch (~> 2.0)
- kramdown (~> 1.14)
+ kramdown (~> 2.1)
+ kramdown-parser-gfm (~> 1.0)
liquid (~> 4.0)
mercenary (~> 0.3.3)
pathutil (~> 0.9)
- rouge (>= 1.7, < 4)
+ rouge (~> 3.0)
safe_yaml (~> 1.0)
+ terminal-table (~> 1.8)
jekyll-multiple-languages (2.0.3)
jekyll-paginate (1.1.0)
- jekyll-sass-converter (1.5.1)
- sass (~> 3.4)
- jekyll-watch (2.0.0)
+ jekyll-sass-converter (2.0.1)
+ sassc (> 2.0.1, < 3.0)
+ jekyll-watch (2.2.1)
listen (~> 3.0)
- json (2.0.4)
- kramdown (1.16.2)
+ json (2.2.0)
+ kramdown (2.1.0)
+ kramdown-parser-gfm (1.1.0)
+ kramdown (~> 2.0)
libv8 (3.16.14.19)
- liquid (4.0.0)
+ liquid (4.0.3)
liquid-c (4.0.0)
liquid (>= 3.0.0)
- listen (3.1.5)
- rb-fsevent (~> 0.9, >= 0.9.4)
- rb-inotify (~> 0.9, >= 0.9.7)
- ruby_dep (~> 1.2)
+ listen (3.2.0)
+ rb-fsevent (~> 0.10, >= 0.10.3)
+ rb-inotify (~> 0.9, >= 0.9.10)
mercenary (0.3.6)
- multipart-post (2.0.0)
- octokit (4.3.0)
- sawyer (~> 0.7.0, >= 0.5.3)
- pathutil (0.16.1)
+ multipart-post (2.1.1)
+ octokit (4.14.0)
+ sawyer (~> 0.8.0, >= 0.5.3)
+ pathutil (0.16.2)
forwardable-extended (~> 2.6)
- rb-fsevent (0.10.2)
- rb-inotify (0.9.10)
- ffi (>= 0.5.0, < 2)
+ public_suffix (4.0.1)
+ rb-fsevent (0.10.3)
+ rb-inotify (0.10.0)
+ ffi (~> 1.0)
ref (2.0.0)
- rouge (3.1.1)
- ruby_dep (1.5.0)
- safe_yaml (1.0.4)
- sass (3.5.5)
- sass-listen (~> 4.0.0)
- sass-listen (4.0.0)
- rb-fsevent (~> 0.9, >= 0.9.4)
- rb-inotify (~> 0.9, >= 0.9.7)
- sawyer (0.7.0)
- addressable (>= 2.3.5, < 2.5)
- faraday (~> 0.8, < 0.10)
- therubyracer (0.12.2)
- libv8 (~> 3.16.14.0)
+ rouge (3.13.0)
+ safe_yaml (1.0.5)
+ sassc (2.2.1)
+ ffi (~> 1.9)
+ sawyer (0.8.2)
+ addressable (>= 2.3.5)
+ faraday (> 0.8, < 2.0)
+ terminal-table (1.8.0)
+ unicode-display_width (~> 1.1, >= 1.1.1)
+ therubyracer (0.12.3)
+ libv8 (~> 3.16.14.15)
ref
+ unicode-display_width (1.6.0)
PLATFORMS
ruby
DEPENDENCIES
- addressable (= 2.4.0)
- hawkins
- jekyll (= 3.7.2)
+ addressable (= 2.7.0)
+ jekyll (= 4.0.0)
jekyll-multiple-languages (= 2.0.3)
jekyll-paginate (= 1.1.0)
- json (= 2.0.4)
+ json (= 2.2.0)
liquid-c (= 4.0.0)
- octokit (~> 4.3.0)
- therubyracer (= 0.12.2)
+ octokit (= 4.14.0)
+ sassc (= 2.2.1)
+ therubyracer (= 0.12.3)
RUBY VERSION
- ruby 2.3.1p112
+ ruby 2.6.3p62
BUNDLED WITH
1.17.2
diff --git a/docs/README.md b/docs/README.md
index 924fcbad55353591711991555c6e2c869346b42a..c3d5f63898d109384aba99003f95c7cad996ecf9 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -42,8 +42,7 @@ If you call the script with the preview flag `build_docs.sh -p`, Jekyll will
start a web server at `localhost:4000` and watch the docs directory for
updates. Use this mode to preview changes locally.
-If you have ruby 2.0 or greater,
-you can call the script with the incremental flag `build_docs.sh -i`.
+You can call the script with the incremental flag `build_docs.sh -i`.
Jekyll will then serve a live preview at `localhost:4000`,
and it will be much faster because it will only rebuild the pages corresponding
to files that are modified. Note that if you are making changes that affect
diff --git a/docs/_config.yml b/docs/_config.yml
index e6e1f8096f595f796e34ccc41a164f9ed6a15f3d..7d4dc7b526c8de5842f93171c9df5cbed4b8939b 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -27,10 +27,10 @@
# we change the version for the complete docs when forking of a release branch
# etc.
# The full version string as referenced in Maven (e.g. 1.2.1)
-version: "1.10-SNAPSHOT"
+version: "1.11-SNAPSHOT"
# For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot
# release this should be the same as the regular version
-version_title: "1.10-SNAPSHOT"
+version_title: "1.11-SNAPSHOT"
# Branch on Github for this version
github_branch: "master"
@@ -64,6 +64,7 @@ is_stable: false
show_outdated_warning: false
previous_docs:
+ 1.10: http://ci.apache.org/projects/flink/flink-docs-release-1.10
1.9: http://ci.apache.org/projects/flink/flink-docs-release-1.9
1.8: http://ci.apache.org/projects/flink/flink-docs-release-1.8
1.7: http://ci.apache.org/projects/flink/flink-docs-release-1.7
@@ -82,6 +83,11 @@ previous_docs:
# to change anything here.
#------------------------------------------------------------------------------
+exclude:
+ - "build_docs.sh"
+ - "build_docs.bat"
+ - "check_links.sh"
+
# Used in some documents to initialize arrays. Don't delete.
array: []
diff --git a/docs/_config_dev_en.yml b/docs/_config_dev_en.yml
index b2854a26731848cf38262d502d69726c0023ee75..a7ca0d0c6d0afe334ee1808a8bb5a58e2e9bf457 100644
--- a/docs/_config_dev_en.yml
+++ b/docs/_config_dev_en.yml
@@ -17,3 +17,9 @@
exclude:
- "*.zh.md"
+ - "build_docs.sh"
+ - "build_docs.bat"
+ - "check_links.sh"
+ - "content"
+ - "content_en"
+ - "content_zh"
diff --git a/docs/_config_dev_zh.yml b/docs/_config_dev_zh.yml
index 4fd38874c2b3e72bf255680e55e518ec4418d2c6..813a6361734a7431baed4d09038a2a191a44ec74 100644
--- a/docs/_config_dev_zh.yml
+++ b/docs/_config_dev_zh.yml
@@ -17,6 +17,12 @@
exclude:
- "*.md"
+ - "build_docs.sh"
+ - "build_docs.bat"
+ - "check_links.sh"
+ - "content"
+ - "content_en"
+ - "content_zh"
include:
- "*.zh.md"
diff --git a/docs/_includes/generated/akka_configuration.html b/docs/_includes/generated/akka_configuration.html
index 83b977507ac962d7ffcc1b2143d4d60f8cde2890..2baeb6ff4f3fdadd2ce0626f89478d47f4652fee 100644
--- a/docs/_includes/generated/akka_configuration.html
+++ b/docs/_includes/generated/akka_configuration.html
@@ -3,123 +3,153 @@
Key
Default
-
Description
+
Type
+
Description
+
+
akka.ask.callstack
+
true
+
Boolean
+
If true, call stack for asynchronous asks are captured. That way, when an ask fails (for example times out), you get a proper exception, describing to the original method call and call site. Note that in case of having millions of concurrent RPC calls, this may add to the memory footprint.
+
akka.ask.timeout
"10 s"
+
String
Timeout used for all futures and blocking Akka calls. If Flink fails due to timeouts then you should try to increase this value. Timeouts can be caused by slow machines or a congested network. The timeout value requires a time-unit specifier (ms/s/min/h/d).
akka.client-socket-worker-pool.pool-size-factor
1.0
+
Double
The pool size factor is used to determine thread pool size using the following formula: ceil(available processors * factor). Resulting size is then bounded by the pool-size-min and pool-size-max values.
akka.client-socket-worker-pool.pool-size-max
2
+
Integer
Max number of threads to cap factor-based number to.
akka.client-socket-worker-pool.pool-size-min
1
+
Integer
Min number of threads to cap factor-based number to.
akka.client.timeout
"60 s"
+
String
Timeout for all blocking calls on the client side.
akka.fork-join-executor.parallelism-factor
2.0
+
Double
The parallelism factor is used to determine thread pool size using the following formula: ceil(available processors * factor). Resulting size is then bounded by the parallelism-min and parallelism-max values.
akka.fork-join-executor.parallelism-max
64
+
Integer
Max number of threads to cap factor-based parallelism number to.
akka.fork-join-executor.parallelism-min
8
+
Integer
Min number of threads to cap factor-based parallelism number to.
akka.framesize
"10485760b"
+
String
Maximum size of messages which are sent between the JobManager and the TaskManagers. If Flink fails because messages exceed this limit, then you should increase it. The message size requires a size-unit specifier.
akka.jvm-exit-on-fatal-error
true
+
Boolean
Exit JVM on fatal Akka errors.
akka.log.lifecycle.events
false
+
Boolean
Turns on the Akka’s remote logging of events. Set this value to 'true' in case of debugging.
akka.lookup.timeout
"10 s"
+
String
Timeout used for the lookup of the JobManager. The timeout value has to contain a time-unit specifier (ms/s/min/h/d).
akka.retry-gate-closed-for
50
+
Long
Milliseconds a gate should be closed for after a remote connection was disconnected.
akka.server-socket-worker-pool.pool-size-factor
1.0
+
Double
The pool size factor is used to determine thread pool size using the following formula: ceil(available processors * factor). Resulting size is then bounded by the pool-size-min and pool-size-max values.
akka.server-socket-worker-pool.pool-size-max
2
+
Integer
Max number of threads to cap factor-based number to.
akka.server-socket-worker-pool.pool-size-min
1
+
Integer
Min number of threads to cap factor-based number to.
akka.ssl.enabled
true
+
Boolean
Turns on SSL for Akka’s remote communication. This is applicable only when the global ssl flag security.ssl.enabled is set to true.
akka.startup-timeout
(none)
+
String
Timeout after which the startup of a remote component is considered being failed.
akka.tcp.timeout
"20 s"
+
String
Timeout for all outbound connections. If you should experience problems with connecting to a TaskManager due to a slow network, you should increase this value.
akka.throughput
15
+
Integer
Number of messages that are processed in a batch before returning the thread to the pool. Low values denote a fair scheduling whereas high values can increase the performance at the cost of unfairness.
akka.transport.heartbeat.interval
"1000 s"
+
String
Heartbeat interval for Akka’s transport failure detector. Since Flink uses TCP, the detector is not necessary. Therefore, the detector is disabled by setting the interval to a very high value. In case you should need the transport failure detector, set the interval to some reasonable value. The interval value requires a time-unit specifier (ms/s/min/h/d).
akka.transport.heartbeat.pause
"6000 s"
+
String
Acceptable heartbeat pause for Akka’s transport failure detector. Since Flink uses TCP, the detector is not necessary. Therefore, the detector is disabled by setting the pause to a very high value. In case you should need the transport failure detector, set the pause to some reasonable value. The pause value requires a time-unit specifier (ms/s/min/h/d).
akka.transport.threshold
300.0
+
Double
Threshold for the transport failure detector. Since Flink uses TCP, the detector is not necessary and, thus, the threshold is set to a high value.
Flag to activate/deactivate bloom filters in the hybrid hash join implementation. In cases where the hash join needs to spill to disk (datasets larger than the reserved fraction of memory), these bloom filters can greatly reduce the number of spilled records, at the cost some CPU cycles.
taskmanager.runtime.max-fan
128
+
Integer
The maximal fan-in for external merge joins and fan-out for spilling hash tables. Limits the number of file handles per operator, but may cause intermediate merging/partitioning, if set too small.
taskmanager.runtime.sort-spilling-threshold
0.8
+
Float
A sort operation starts spilling when this fraction of its memory budget is full.
diff --git a/docs/_includes/generated/all_jobmanager_section.html b/docs/_includes/generated/all_jobmanager_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..ab01a856e032db1fd9acf753447b4c8b58dea8b0
--- /dev/null
+++ b/docs/_includes/generated/all_jobmanager_section.html
@@ -0,0 +1,66 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
jobmanager.archive.fs.dir
+
(none)
+
String
+
Dictionary for JobManager to store the archives of completed jobs.
+
+
+
jobmanager.execution.attempts-history-size
+
16
+
Integer
+
The maximum number of prior execution attempts kept in history.
+
+
+
jobmanager.execution.failover-strategy
+
region
+
String
+
This option specifies how the job computation recovers from task failures. Accepted values are:
'full': Restarts all tasks to recover the job.
'region': Restarts all tasks that could be affected by the task failure. More details can be found here.
+
+
+
jobmanager.heap.size
+
"1024m"
+
String
+
JVM heap size for the JobManager.
+
+
+
jobmanager.rpc.address
+
(none)
+
String
+
The config parameter defining the network address to connect to for communication with the job manager. This value is only interpreted in setups where a single JobManager with static name or address exists (simple standalone setups, or container setups with dynamic service name resolution). It is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby JobManagers.
+
+
+
jobmanager.rpc.port
+
6123
+
Integer
+
The config parameter defining the network port to connect to for communication with the job manager. Like jobmanager.rpc.address, this value is only interpreted in setups where a single JobManager with static name/address and port exists (simple standalone setups, or container setups with dynamic service name resolution). This config option is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby JobManagers.
+
+
+
jobstore.cache-size
+
52428800
+
Long
+
The job store cache size in bytes which is used to keep completed jobs in memory.
+
+
+
jobstore.expiration-time
+
3600
+
Long
+
The time in seconds after which a completed job expires and is purged from the job store.
+
+
+
jobstore.max-capacity
+
2147483647
+
Integer
+
The max number of completed jobs that can be kept in the job store.
+
+
+
diff --git a/docs/_includes/generated/all_taskmanager_network_section.html b/docs/_includes/generated/all_taskmanager_network_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..04a85662bae4a5305c1c1587dad721d36f643dd8
--- /dev/null
+++ b/docs/_includes/generated/all_taskmanager_network_section.html
@@ -0,0 +1,96 @@
+
Boolean flag indicating whether the shuffle data will be compressed for blocking shuffle mode. Note that data is compressed per buffer and compression can incur extra CPU overhead, so it is more effective for IO bounded scenario when data compression ratio is high. Currently, shuffle data compression is an experimental feature and the config option can be changed in the future.
+
+
+
taskmanager.network.blocking-shuffle.type
+
"file"
+
String
+
The blocking shuffle type, either "mmap" or "file". The "auto" means selecting the property type automatically based on system memory architecture (64 bit for mmap and 32 bit for file). Note that the memory usage of mmap is not accounted by configured memory limits, but some resource frameworks like yarn would track this memory usage and kill the container once memory exceeding some threshold. Also note that this option is experimental and might be changed future.
+
+
+
taskmanager.network.detailed-metrics
+
false
+
Boolean
+
Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths.
+
+
+
taskmanager.network.memory.buffers-per-channel
+
2
+
Integer
+
Maximum number of network buffers to use for each outgoing/incoming channel (subpartition/input channel).In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization.
Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster.
The Netty send and receive buffer size. This defaults to the system buffer size (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.
+
+
+
taskmanager.network.netty.server.backlog
+
0
+
Integer
+
The netty server connection backlog.
+
+
+
taskmanager.network.netty.server.numThreads
+
-1
+
Integer
+
The number of Netty server threads.
+
+
+
taskmanager.network.netty.transport
+
"auto"
+
String
+
The Netty transport type, either "nio" or "epoll". The "auto" means selecting the property mode automatically based on the platform. Note that the "epoll" mode can get better performance, less GC and have more advanced features which are only available on modern Linux.
+
+
+
taskmanager.network.request-backoff.initial
+
100
+
Integer
+
Minimum backoff in milliseconds for partition requests of input channels.
+
+
+
taskmanager.network.request-backoff.max
+
10000
+
Integer
+
Maximum backoff in milliseconds for partition requests of input channels.
+
+
+
diff --git a/docs/_includes/generated/all_taskmanager_section.html b/docs/_includes/generated/all_taskmanager_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..692c38b6e22f659a0aa5f1081cc3af524696a776
--- /dev/null
+++ b/docs/_includes/generated/all_taskmanager_section.html
@@ -0,0 +1,115 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
task.cancellation.interval
+
30000
+
Long
+
Time interval between two successive task cancellation attempts in milliseconds.
+
+
+
task.cancellation.timeout
+
180000
+
Long
+
Timeout in milliseconds after which a task cancellation times out and leads to a fatal TaskManager error. A value of 0 deactivates the watch dog.
+
+
+
task.cancellation.timers.timeout
+
7500
+
Long
+
Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.
+
+
+
taskmanager.data.port
+
0
+
Integer
+
The task manager’s port used for data exchange operations.
+
+
+
taskmanager.data.ssl.enabled
+
true
+
Boolean
+
Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true
+
+
+
taskmanager.debug.memory.log
+
false
+
Boolean
+
Flag indicating whether to start a thread, which repeatedly logs the memory usage of the JVM.
+
+
+
taskmanager.debug.memory.log-interval
+
5000
+
Long
+
The interval (in ms) for the log thread to log the current memory usage.
+
+
+
taskmanager.host
+
(none)
+
String
+
The address of the network interface that the TaskManager binds to. This option can be used to define explicitly a binding address. Because different TaskManagers need different values for this option, usually it is specified in an additional non-shared TaskManager-specific config file.
+
+
+
taskmanager.jvm-exit-on-oom
+
false
+
Boolean
+
Whether to kill the TaskManager when the task thread throws an OutOfMemoryError.
+
+
+
taskmanager.memory.segment-size
+
32 kb
+
MemorySize
+
Size of memory buffers used by the network stack and the memory manager.
+
+
+
taskmanager.network.bind-policy
+
"ip"
+
String
+
The automatic address binding policy used by the TaskManager if "taskmanager.host" is not set. The value should be one of the following:
+
"name" - uses hostname as binding address
"ip" - uses host's ip address as binding address
+
+
+
taskmanager.numberOfTaskSlots
+
1
+
Integer
+
The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores).
+
+
+
taskmanager.registration.initial-backoff
+
500 ms
+
Duration
+
The initial registration backoff between two consecutive registration attempts. The backoff is doubled for each new registration attempt until it reaches the maximum registration backoff.
+
+
+
taskmanager.registration.max-backoff
+
30 s
+
Duration
+
The maximum registration backoff between two consecutive registration attempts. The max registration backoff requires a time unit specifier (ms/s/min/h/d).
+
+
+
taskmanager.registration.refused-backoff
+
10 s
+
Duration
+
The backoff after a registration has been refused by the job manager before retrying to connect.
+
+
+
taskmanager.registration.timeout
+
5 min
+
Duration
+
Defines the timeout for the TaskManager registration. If the duration is exceeded without a successful registration, then the TaskManager terminates.
+
+
+
taskmanager.rpc.port
+
"0"
+
String
+
The task manager’s IPC port. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple TaskManagers are running on the same machine.
The state backend to be used to store and checkpoint state.
state.backend.async
true
+
Boolean
Option whether the state backend should use an asynchronous snapshot method where possible and configurable. Some state backends may not support asynchronous snapshots, or only support asynchronous snapshots, and ignore this option.
state.backend.fs.memory-threshold
1024
+
Integer
The minimum size of state data files. All state chunks smaller than that are stored inline in the root checkpoint metadata file.
state.backend.fs.write-buffer-size
4096
+
Integer
The default size of the write buffer for the checkpoint streams that write to file systems. The actual write buffer size is determined to be the maximum of the value of this option and option 'state.backend.fs.memory-threshold'.
state.backend.incremental
false
+
Boolean
Option whether the state backend should create incremental checkpoints, if possible. For an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the complete checkpoint state. Some state backends may not support incremental checkpoints and ignore this option.
state.backend.local-recovery
false
+
Boolean
This option configures local recovery for this state backend. By default, local recovery is deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option.
state.checkpoints.dir
(none)
+
String
The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).
state.checkpoints.num-retained
1
+
Integer
The maximum number of completed checkpoints to retain.
state.savepoints.dir
(none)
+
String
The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).
taskmanager.state.local.root-dirs
(none)
+
String
The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option
Enable the slot spread out allocation strategy. This strategy tries to spread out the slots evenly across all available `TaskExecutors`.
+
cluster.registration.error-delay
10000
+
Long
The pause made after an registration attempt caused an exception (other than timeout) in milliseconds.
cluster.registration.initial-timeout
100
+
Long
Initial registration timeout between cluster components in milliseconds.
cluster.registration.max-timeout
30000
+
Long
Maximum registration timeout between cluster components in milliseconds.
cluster.registration.refused-registration-delay
30000
+
Long
The pause made after the registration attempt was refused in milliseconds.
cluster.services.shutdown-timeout
30000
+
Long
The shutdown timeout for cluster services like executors in milliseconds.
diff --git a/docs/_includes/generated/common_high_availability_section.html b/docs/_includes/generated/common_high_availability_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..4c06ca778cf2ded408bf5ca0cc8937aaba20704d
--- /dev/null
+++ b/docs/_includes/generated/common_high_availability_section.html
@@ -0,0 +1,30 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
high-availability
+
"NONE"
+
String
+
Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER" or specify FQN of factory class.
+
+
+
high-availability.cluster-id
+
"/default"
+
String
+
The ID of the Flink cluster, used to separate multiple Flink clusters from each other. Needs to be set for standalone clusters but is automatically inferred in YARN and Mesos.
+
+
+
high-availability.storageDir
+
(none)
+
String
+
File system path (URI) where Flink persists metadata in high-availability setups.
+
+
+
diff --git a/docs/_includes/generated/common_high_availability_zk_section.html b/docs/_includes/generated/common_high_availability_zk_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..18175fb7339c96c8145102ed4fae49e480981608
--- /dev/null
+++ b/docs/_includes/generated/common_high_availability_zk_section.html
@@ -0,0 +1,24 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
high-availability.zookeeper.path.root
+
"/flink"
+
String
+
The root path under which Flink stores its entries in ZooKeeper.
+
+
+
high-availability.zookeeper.quorum
+
(none)
+
String
+
The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.
+
+
+
diff --git a/docs/_includes/generated/common_host_port_section.html b/docs/_includes/generated/common_host_port_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..5deb71edc484fda84e711183ebb3d00055815081
--- /dev/null
+++ b/docs/_includes/generated/common_host_port_section.html
@@ -0,0 +1,72 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
jobmanager.rpc.address
+
(none)
+
String
+
The config parameter defining the network address to connect to for communication with the job manager. This value is only interpreted in setups where a single JobManager with static name or address exists (simple standalone setups, or container setups with dynamic service name resolution). It is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby JobManagers.
+
+
+
jobmanager.rpc.port
+
6123
+
Integer
+
The config parameter defining the network port to connect to for communication with the job manager. Like jobmanager.rpc.address, this value is only interpreted in setups where a single JobManager with static name/address and port exists (simple standalone setups, or container setups with dynamic service name resolution). This config option is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby JobManagers.
+
+
+
metrics.internal.query-service.port
+
"0"
+
String
+
The port range used for Flink's internal metric query service. Accepts a list of ports (“50100,50101”), ranges(“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple Flink components are running on the same machine. Per default Flink will pick a random port.
+
+
+
rest.address
+
(none)
+
String
+
The address that should be used by clients to connect to the server.
+
+
+
rest.bind-address
+
(none)
+
String
+
The address that the server binds itself.
+
+
+
rest.bind-port
+
"8081"
+
String
+
The port that the server binds itself. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple Rest servers are running on the same machine.
+
+
+
rest.port
+
8081
+
Integer
+
The port that the client connects to. If rest.bind-port has not been specified, then the REST server will bind to this port.
+
+
+
taskmanager.data.port
+
0
+
Integer
+
The task manager’s port used for data exchange operations.
+
+
+
taskmanager.host
+
(none)
+
String
+
The address of the network interface that the TaskManager binds to. This option can be used to define explicitly a binding address. Because different TaskManagers need different values for this option, usually it is specified in an additional non-shared TaskManager-specific config file.
+
+
+
taskmanager.rpc.port
+
"0"
+
String
+
The task manager’s IPC port. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple TaskManagers are running on the same machine.
+
+
+
diff --git a/docs/_includes/generated/common_memory_section.html b/docs/_includes/generated/common_memory_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..713edff3b253b979073723b895fbb0c0397be307
--- /dev/null
+++ b/docs/_includes/generated/common_memory_section.html
@@ -0,0 +1,102 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
taskmanager.memory.flink.size
+
(none)
+
MemorySize
+
Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Network Memory. See also 'taskmanager.memory.process.size' for total process memory size configuration.
+
+
+
taskmanager.memory.framework.heap.size
+
128 mb
+
MemorySize
+
Framework Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for TaskExecutor framework, which will not be allocated to task slots.
+
+
+
taskmanager.memory.framework.off-heap.size
+
128 mb
+
MemorySize
+
Framework Off-Heap Memory size for TaskExecutors. This is the size of off-heap memory (JVM direct memory and native memory) reserved for TaskExecutor framework, which will not be allocated to task slots. The configured value will be fully counted when Flink calculates the JVM max direct memory size parameter.
+
+
+
taskmanager.memory.jvm-metaspace.size
+
96 mb
+
MemorySize
+
JVM Metaspace Size for the TaskExecutors.
+
+
+
taskmanager.memory.jvm-overhead.fraction
+
0.1
+
Float
+
Fraction of Total Process Memory to be reserved for JVM Overhead. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
+
+
+
taskmanager.memory.jvm-overhead.max
+
1 gb
+
MemorySize
+
Max JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
+
+
+
taskmanager.memory.jvm-overhead.min
+
192 mb
+
MemorySize
+
Min JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
+
+
+
taskmanager.memory.managed.fraction
+
0.4
+
Float
+
Fraction of Total Flink Memory to be used as Managed Memory, if Managed Memory size is not explicitly specified.
+
+
+
taskmanager.memory.managed.size
+
(none)
+
MemorySize
+
Managed Memory size for TaskExecutors. This is the size of off-heap memory managed by the memory manager, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backend. Memory consumers can either allocate memory from the memory manager in the form of MemorySegments, or reserve bytes from the memory manager and keep their memory usage within that boundary. If unspecified, it will be derived to make up the configured fraction of the Total Flink Memory.
+
+
+
taskmanager.memory.network.fraction
+
0.1
+
Float
+
Fraction of Total Flink Memory to be used as Network Memory. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max size to the same value.
+
+
+
taskmanager.memory.network.max
+
1 gb
+
MemorySize
+
Max Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max to the same value.
+
+
+
taskmanager.memory.network.min
+
64 mb
+
MemorySize
+
Min Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max to the same value.
+
+
+
taskmanager.memory.process.size
+
(none)
+
MemorySize
+
Total Process Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. On containerized setups, this should be set to the container memory. See also 'taskmanager.memory.flink.size' for total Flink memory size configuration.
+
+
+
taskmanager.memory.task.heap.size
+
(none)
+
MemorySize
+
Task Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for tasks. If not specified, it will be derived as Total Flink Memory minus Framework Heap Memory, Task Off-Heap Memory, Managed Memory and Network Memory.
+
+
+
taskmanager.memory.task.off-heap.size
+
0 bytes
+
MemorySize
+
Task Off-Heap Memory size for TaskExecutors. This is the size of off heap memory (JVM direct memory and native memory) reserved for tasks. The configured value will be fully counted when Flink calculates the JVM max direct memory size parameter.
+
+
+
diff --git a/docs/_includes/generated/common_miscellaneous_section.html b/docs/_includes/generated/common_miscellaneous_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..0a4e15af8fd2d250feb7c14668eae4b5003e5fc1
--- /dev/null
+++ b/docs/_includes/generated/common_miscellaneous_section.html
@@ -0,0 +1,24 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
fs.default-scheme
+
(none)
+
String
+
The default filesystem scheme, used for paths that do not declare a scheme explicitly. May contain an authority, e.g. host:port in case of an HDFS NameNode.
+
+
+
io.tmp.dirs
+
'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.
+
String
+
Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.
+
+
+
diff --git a/docs/_includes/generated/common_section.html b/docs/_includes/generated/common_state_backends_section.html
similarity index 39%
rename from docs/_includes/generated/common_section.html
rename to docs/_includes/generated/common_state_backends_section.html
index 8d7a232446181bc1870292f23c6543fcd0ec8c87..0df38e9e763348c249d10feddbbe406ba576fbbb 100644
--- a/docs/_includes/generated/common_section.html
+++ b/docs/_includes/generated/common_state_backends_section.html
@@ -3,64 +3,52 @@
Key
Default
-
Description
+
Type
+
Description
-
-
jobmanager.heap.size
-
"1024m"
-
JVM heap size for the JobManager.
-
-
-
taskmanager.memory.total-flink.size
-
(none)
-
Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Shuffle Memory.
-
-
-
parallelism.default
-
1
-
Default parallelism for jobs.
-
-
-
taskmanager.numberOfTaskSlots
-
1
-
The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores).
-
state.backend
(none)
+
String
The state backend to be used to store and checkpoint state.
state.checkpoints.dir
(none)
+
String
The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).
state.savepoints.dir
(none)
+
String
The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).
-
high-availability
-
"NONE"
-
Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER" or specify FQN of factory class.
+
state.backend.incremental
+
false
+
Boolean
+
Option whether the state backend should create incremental checkpoints, if possible. For an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the complete checkpoint state. Some state backends may not support incremental checkpoints and ignore this option.
-
high-availability.storageDir
-
(none)
-
File system path (URI) where Flink persists metadata in high-availability setups.
+
state.backend.local-recovery
+
false
+
Boolean
+
This option configures local recovery for this state backend. By default, local recovery is deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option.
-
security.ssl.internal.enabled
-
false
-
Turns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc).
+
state.checkpoints.num-retained
+
1
+
Integer
+
The maximum number of completed checkpoints to retain.
-
security.ssl.rest.enabled
-
false
-
Turns on SSL for external communication via the REST endpoints.
+
taskmanager.state.local.root-dirs
+
(none)
+
String
+
The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option
A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. These patterns are appended to "classloader.parent-first-patterns.default".
A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. This setting should generally not be modified. To add another pattern we recommend to use "classloader.parent-first-patterns.additional" instead.
classloader.resolve-order
"child-first"
+
String
Defines the class resolution strategy when loading classes from user code, meaning whether to first check the user code jar ("child-first") or the application classpath ("parent-first"). The default settings indicate to load classes first from the user code jar, which means that user code jars can include and load different dependencies than Flink uses (transitively).
+
+
fs.default-scheme
+
(none)
+
String
+
The default filesystem scheme, used for paths that do not declare a scheme explicitly. May contain an authority, e.g. host:port in case of an HDFS NameNode.
+
+
+
fs.output.always-create-directory
+
false
+
Boolean
+
File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to "true", writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to "false", the writer will directly create the file directly at the output path, without creating a containing directory.
+
+
+
fs.overwrite-files
+
false
+
Boolean
+
Specifies whether file output writers should overwrite existing files by default. Set to "true" to overwrite by default,"false" otherwise.
+
io.tmp.dirs
'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.
+
String
Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.
parallelism.default
1
+
Integer
Default parallelism for jobs.
diff --git a/docs/_includes/generated/deployment_configuration.html b/docs/_includes/generated/deployment_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..325dc982012b2e500cbc8676ed8373cb9d1b230f
--- /dev/null
+++ b/docs/_includes/generated/deployment_configuration.html
@@ -0,0 +1,30 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
execution.attached
+
false
+
Boolean
+
Specifies if the pipeline is submitted in attached or detached mode.
+
+
+
execution.shutdown-on-attached-exit
+
false
+
Boolean
+
If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated abruptly, e.g., in response to a user interrupt, such as typing Ctrl + C.
+
+
+
execution.target
+
(none)
+
String
+
The deployment target for the execution, e.g. "local" for local execution.
+
+
+
diff --git a/docs/_includes/generated/deprecated_file_sinks_section.html b/docs/_includes/generated/deprecated_file_sinks_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..26e3d5385648f582f432e7c76453e63fb0bd21a1
--- /dev/null
+++ b/docs/_includes/generated/deprecated_file_sinks_section.html
@@ -0,0 +1,24 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
fs.output.always-create-directory
+
false
+
Boolean
+
File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to "true", writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to "false", the writer will directly create the file directly at the output path, without creating a containing directory.
+
+
+
fs.overwrite-files
+
false
+
Boolean
+
Specifies whether file output writers should overwrite existing files by default. Set to "true" to overwrite by default,"false" otherwise.
Path to hadoop configuration directory. It is required to read HDFS and/or YARN configuration. You can also set it via environment variable.
env.java.opts
(none)
+
String
Java options to start the JVM of all Flink processes with.
env.java.opts.historyserver
(none)
+
String
Java options to start the JVM of the HistoryServer with.
env.java.opts.jobmanager
(none)
+
String
Java options to start the JVM of the JobManager with.
env.java.opts.taskmanager
(none)
+
String
Java options to start the JVM of the TaskManager with.
env.log.dir
(none)
+
String
Defines the directory where the Flink logs are saved. It has to be an absolute path. (Defaults to the log directory under Flink’s home)
env.log.max
5
+
Integer
The maximum number of old log files to keep.
env.ssh.opts
(none)
+
String
Additional command line options passed to SSH clients when starting or stopping JobManager, TaskManager, and Zookeeper services (start-cluster.sh, stop-cluster.sh, start-zookeeper-quorum.sh, stop-zookeeper-quorum.sh).
env.yarn.conf.dir
(none)
+
String
Path to yarn configuration directory. It is required to run flink on YARN. You can also set it via environment variable.
diff --git a/docs/_includes/generated/execution_checkpointing_configuration.html b/docs/_includes/generated/execution_checkpointing_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..59a84f45fd74d29523a191df7d29bedb164991f6
--- /dev/null
+++ b/docs/_includes/generated/execution_checkpointing_configuration.html
@@ -0,0 +1,60 @@
+
Possible values: [DELETE_ON_CANCELLATION, RETAIN_ON_CANCELLATION]
+
Externalized checkpoints write their meta data out to persistent storage and are not automatically cleaned up when the owning job fails or is suspended (terminating with job status `JobStatus#FAILED` or `JobStatus#SUSPENDED`. In this case, you have to manually clean up the checkpoint state, both the meta data and actual program state.
The mode defines how an externalized checkpoint should be cleaned up on job cancellation. If you choose to retain externalized checkpoints on cancellation you have to handle checkpoint clean up manually when you cancel the job as well (terminating with job status `JobStatus#CANCELED`).
The target directory for externalized checkpoints is configured via `state.checkpoints.dir`.
+
+
+
execution.checkpointing.interval
+
(none)
+
Duration
+
Gets the interval in which checkpoints are periodically scheduled.
This setting defines the base interval. Checkpoint triggering may be delayed by the settings `execution.checkpointing.max-concurrent-checkpoints` and `execution.checkpointing.min-pause`
The maximum number of checkpoint attempts that may be in progress at the same time. If this value is n, then no checkpoints will be triggered while n checkpoint attempts are currently in flight. For the next checkpoint to be triggered, one checkpoint attempt would need to finish or expire.
+
+
+
execution.checkpointing.min-pause
+
0 ms
+
Duration
+
The minimal pause between checkpointing attempts. This setting defines how soon thecheckpoint coordinator may trigger another checkpoint after it becomes possible to triggeranother checkpoint with respect to the maximum number of concurrent checkpoints(see `execution.checkpointing.max-concurrent-checkpoints`).
If the maximum number of concurrent checkpoints is set to one, this setting makes effectively sure that a minimum amount of time passes where no checkpoint is in progress at all.
+
+
+
execution.checkpointing.mode
+
EXACTLY_ONCE
+
Enum
Possible values: [EXACTLY_ONCE, AT_LEAST_ONCE]
+
The checkpointing mode (exactly-once vs. at-least-once).
The max number of async i/o operation that the async lookup join can trigger.
table.exec.async-lookup.timeout
BatchStreaming
"3 min"
+
String
The async timeout for the asynchronous operation to complete.
table.exec.disabled-operators
Batch
(none)
+
String
Mainly for testing. A comma-separated list of operator names, each name represents a kind of disabled operator.
Operators that can be disabled include "NestedLoopJoin", "ShuffleHashJoin", "BroadcastHashJoin", "SortMergeJoin", "HashAgg", "SortAgg".
By default no operator is disabled.
@@ -27,46 +31,31 @@ By default no operator is disabled.
table.exec.mini-batch.allow-latency
Streaming
"-1 ms"
+
String
The maximum latency can be used for MiniBatch to buffer input records. MiniBatch is an optimization to buffer input records to reduce state access. MiniBatch is triggered with the allowed latency interval and when the maximum number of buffered records reached. NOTE: If table.exec.mini-batch.enabled is set true, its value must be greater than zero.
table.exec.mini-batch.enabled
Streaming
false
+
Boolean
Specifies whether to enable MiniBatch optimization. MiniBatch is an optimization to buffer input records to reduce state access. This is disabled by default. To enable this, users should set this config to true. NOTE: If mini-batch is enabled, 'table.exec.mini-batch.allow-latency' and 'table.exec.mini-batch.size' must be set.
table.exec.mini-batch.size
Streaming
-1
+
Long
The maximum number of input records can be buffered for MiniBatch. MiniBatch is an optimization to buffer input records to reduce state access. MiniBatch is triggered with the allowed latency interval and when the maximum number of buffered records reached. NOTE: MiniBatch only works for non-windowed aggregations currently. If table.exec.mini-batch.enabled is set true, its value must be positive.
table.exec.resource.default-parallelism
BatchStreaming
-1
+
Integer
Sets default parallelism for all operators (such as aggregate, join, filter) to run with parallel instances. This config has a higher priority than parallelism of StreamExecutionEnvironment (actually, this config overrides the parallelism of StreamExecutionEnvironment). A value of -1 indicates that no default parallelism is set, then it will fallback to use the parallelism of StreamExecutionEnvironment.
-
-
table.exec.resource.external-buffer-memory
Batch
-
"10 mb"
-
Sets the external buffer memory size that is used in sort merge join and nested join and over window.
-
-
-
table.exec.resource.hash-agg.memory
Batch
-
"128 mb"
-
Sets the managed memory size of hash aggregate operator.
-
-
-
table.exec.resource.hash-join.memory
Batch
-
"128 mb"
-
Sets the managed memory for hash join operator. It defines the lower limit.
-
-
-
table.exec.resource.sort.memory
Batch
-
"128 mb"
-
Sets the managed buffer memory size for sort operator.
-
table.exec.shuffle-mode
Batch
"batch"
+
String
Sets exec shuffle mode. Only batch or pipeline can be set.
batch: the job will run stage by stage.
pipeline: the job will run in streaming mode, but it may cause resource deadlock that receiver waits for resource to start when the sender holds resource to wait to send data to the receiver.
@@ -74,36 +63,43 @@ pipeline: the job will run in streaming mode, but it may cause resource deadlock
table.exec.sort.async-merge-enabled
Batch
true
+
Boolean
Whether to asynchronously merge sorted spill files.
table.exec.sort.default-limit
Batch
-1
+
Integer
Default limit when user don't set a limit after order by. -1 indicates that this configuration is ignored.
table.exec.sort.max-num-file-handles
Batch
128
+
Integer
The maximal fan-in for external merge sort. It limits the number of file handles per operator. If it is too small, may cause intermediate merging. But if it is too large, it will cause too many files opened at the same time, consume memory and lead to random reading.
table.exec.source.idle-timeout
Streaming
"-1 ms"
+
String
When a source do not receive any elements for the timeout time, it will be marked as temporarily idle. This allows downstream tasks to advance their watermarks without the need to wait for watermarks from this source while it is idle.
table.exec.spill-compression.block-size
Batch
"64 kb"
+
String
The memory size used to do compress when spilling data. The larger the memory, the higher the compression ratio, but more memory resource will be consumed by the job.
table.exec.spill-compression.enabled
Batch
true
+
Boolean
Whether to compress spilled data. Currently we only support compress spilled data for sort and hash-agg and hash-join operators.
table.exec.window-agg.buffer-size-limit
Batch
100000
+
Integer
Sets the window elements buffer size limit used in group window agg operator.
diff --git a/docs/_includes/generated/execution_configuration.html b/docs/_includes/generated/execution_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..037960cdcb399007873b72adfbac41ab6a849128
--- /dev/null
+++ b/docs/_includes/generated/execution_configuration.html
@@ -0,0 +1,24 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
execution.buffer-timeout
+
100 ms
+
Duration
+
The maximum time frequency (milliseconds) for the flushing of the output buffers. By default the output buffers flush frequently to provide low latency and to aid smooth developer experience. Setting the parameter can result in three logical modes:
A positive value triggers flushing periodically by that interval
0 triggers flushing after every record thus minimizing latency
-1 ms triggers flushing only when the output buffer is full thus maximizing throughput
+
+
+
execution.checkpointing.snapshot-compression
+
false
+
Boolean
+
Tells if we should use compression for the state snapshot data or not
+
+
+
diff --git a/docs/_includes/generated/expert_class_loading_section.html b/docs/_includes/generated/expert_class_loading_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..1732e846227d76e556ba4fd1a7ed44e201ef2a72
--- /dev/null
+++ b/docs/_includes/generated/expert_class_loading_section.html
@@ -0,0 +1,30 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
classloader.parent-first-patterns.additional
+
(none)
+
String
+
A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. These patterns are appended to "classloader.parent-first-patterns.default".
A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against the fully qualified class name. This setting should generally not be modified. To add another pattern we recommend to use "classloader.parent-first-patterns.additional" instead.
+
+
+
classloader.resolve-order
+
"child-first"
+
String
+
Defines the class resolution strategy when loading classes from user code, meaning whether to first check the user code jar ("child-first") or the application classpath ("parent-first"). The default settings indicate to load classes first from the user code jar, which means that user code jars can include and load different dependencies than Flink uses (transitively).
+
+
+
diff --git a/docs/_includes/generated/expert_fault_tolerance_section.html b/docs/_includes/generated/expert_fault_tolerance_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..520717b026c744b470e97544c1165c77e9719d55
--- /dev/null
+++ b/docs/_includes/generated/expert_fault_tolerance_section.html
@@ -0,0 +1,60 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
cluster.registration.error-delay
+
10000
+
Long
+
The pause made after an registration attempt caused an exception (other than timeout) in milliseconds.
+
+
+
cluster.registration.initial-timeout
+
100
+
Long
+
Initial registration timeout between cluster components in milliseconds.
+
+
+
cluster.registration.max-timeout
+
30000
+
Long
+
Maximum registration timeout between cluster components in milliseconds.
+
+
+
cluster.registration.refused-registration-delay
+
30000
+
Long
+
The pause made after the registration attempt was refused in milliseconds.
+
+
+
cluster.services.shutdown-timeout
+
30000
+
Long
+
The shutdown timeout for cluster services like executors in milliseconds.
+
+
+
heartbeat.interval
+
10000
+
Long
+
Time interval for requesting heartbeat from sender side.
+
+
+
heartbeat.timeout
+
50000
+
Long
+
Timeout for requesting and receiving heartbeat for both sender and receiver sides.
+
+
+
jobmanager.execution.failover-strategy
+
region
+
String
+
This option specifies how the job computation recovers from task failures. Accepted values are:
'full': Restarts all tasks to recover the job.
'region': Restarts all tasks that could be affected by the task failure. More details can be found here.
+
+
+
diff --git a/docs/_includes/generated/expert_high_availability_section.html b/docs/_includes/generated/expert_high_availability_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..4a571e70cab789927f5f8cb0c0b0e0f2eaacc3ab
--- /dev/null
+++ b/docs/_includes/generated/expert_high_availability_section.html
@@ -0,0 +1,18 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
high-availability.jobmanager.port
+
"0"
+
String
+
The port (range) used by the Flink Master for its RPC connections in highly-available setups. In highly-available setups, this value is used instead of 'jobmanager.rpc.port'.A value of '0' means that a random free port is chosen. TaskManagers discover this port through the high-availability services (leader election), so a random port or a port range works without requiring any additional means of service discovery.
+
+
+
diff --git a/docs/_includes/generated/expert_high_availability_zk_section.html b/docs/_includes/generated/expert_high_availability_zk_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..d7774e22560e961596615780db107c913f4fd9c1
--- /dev/null
+++ b/docs/_includes/generated/expert_high_availability_zk_section.html
@@ -0,0 +1,84 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
high-availability.zookeeper.client.acl
+
"open"
+
String
+
Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).
ZooKeeper root path (ZNode) for checkpoint counters.
+
+
+
high-availability.zookeeper.path.checkpoints
+
"/checkpoints"
+
String
+
ZooKeeper root path (ZNode) for completed checkpoints.
+
+
+
high-availability.zookeeper.path.jobgraphs
+
"/jobgraphs"
+
String
+
ZooKeeper root path (ZNode) for job graphs
+
+
+
high-availability.zookeeper.path.latch
+
"/leaderlatch"
+
String
+
Defines the znode of the leader latch which is used to elect the leader.
+
+
+
high-availability.zookeeper.path.leader
+
"/leader"
+
String
+
Defines the znode of the leader which contains the URL to the leader and the current leader session ID.
+
+
+
high-availability.zookeeper.path.mesos-workers
+
"/mesos-workers"
+
String
+
The ZooKeeper root path for persisting the Mesos worker information.
+
+
+
high-availability.zookeeper.path.running-registry
+
"/running_job_registry/"
+
String
+
+
+
+
diff --git a/docs/_includes/generated/expert_rest_section.html b/docs/_includes/generated/expert_rest_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..ab1a70cb67e992723dd371e3fdc9f68a733ffd57
--- /dev/null
+++ b/docs/_includes/generated/expert_rest_section.html
@@ -0,0 +1,66 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
rest.await-leader-timeout
+
30000
+
Long
+
The time in ms that the client waits for the leader address, e.g., Dispatcher or WebMonitorEndpoint
+
+
+
rest.client.max-content-length
+
104857600
+
Integer
+
The maximum content length in bytes that the client will handle.
+
+
+
rest.connection-timeout
+
15000
+
Long
+
The maximum time in ms for the client to establish a TCP connection.
+
+
+
rest.idleness-timeout
+
300000
+
Long
+
The maximum time in ms for a connection to stay idle before failing.
+
+
+
rest.retry.delay
+
3000
+
Long
+
The time in ms that the client waits between retries (See also `rest.retry.max-attempts`).
+
+
+
rest.retry.max-attempts
+
20
+
Integer
+
The number of retries the client will attempt if a retryable operations fails.
+
+
+
rest.server.max-content-length
+
104857600
+
Integer
+
The maximum content length in bytes that the server will handle.
+
+
+
rest.server.numThreads
+
4
+
Integer
+
The number of threads for the asynchronous processing of requests.
+
+
+
rest.server.thread-priority
+
5
+
Integer
+
Thread priority of the REST server's executor for processing asynchronous requests. Lowering the thread priority will give Flink's main components more CPU time whereas increasing will allocate more time for the REST server's processing.
+
+
+
diff --git a/docs/_includes/generated/expert_rocksdb_section.html b/docs/_includes/generated/expert_rocksdb_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..d3cbfe16647b8778be1f9d8bf075e174a19f1e24
--- /dev/null
+++ b/docs/_includes/generated/expert_rocksdb_section.html
@@ -0,0 +1,36 @@
+
The options factory class for RocksDB to create DBOptions and ColumnFamilyOptions. The default options factory is org.apache.flink.contrib.streaming.state.DefaultConfigurableOptionsFactory, and it would read the configured options which provided in 'RocksDBConfigurableOptions'.
+
+
+
state.backend.rocksdb.predefined-options
+
"DEFAULT"
+
String
+
The predefined settings for RocksDB DBOptions and ColumnFamilyOptions by Flink community. Current supported candidate predefined-options are DEFAULT, SPINNING_DISK_OPTIMIZED, SPINNING_DISK_OPTIMIZED_HIGH_MEM or FLASH_SSD_OPTIMIZED. Note that user customized options and options from the OptionsFactory are applied on top of these predefined ones.
+
+
+
diff --git a/docs/_includes/generated/expert_scheduling_section.html b/docs/_includes/generated/expert_scheduling_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..a29ad69c76e6c96bc2995314fcea4e180651c4fb
--- /dev/null
+++ b/docs/_includes/generated/expert_scheduling_section.html
@@ -0,0 +1,30 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
cluster.evenly-spread-out-slots
+
false
+
Boolean
+
Enable the slot spread out allocation strategy. This strategy tries to spread out the slots evenly across all available `TaskExecutors`.
+
+
+
slot.idle.timeout
+
50000
+
Long
+
The timeout in milliseconds for a idle slot in Slot Pool.
+
+
+
slot.request.timeout
+
300000
+
Long
+
The timeout in milliseconds for requesting a slot from Slot Pool.
+
+
+
diff --git a/docs/_includes/generated/expert_security_ssl_section.html b/docs/_includes/generated/expert_security_ssl_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..9e5af5f622449802b14d5f30936646ea5264ae2a
--- /dev/null
+++ b/docs/_includes/generated/expert_security_ssl_section.html
@@ -0,0 +1,42 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
security.ssl.internal.close-notify-flush-timeout
+
-1
+
Integer
+
The timeout (in ms) for flushing the `close_notify` that was triggered by closing a channel. If the `close_notify` was not flushed in the given timeout the channel will be closed forcibly. (-1 = use system default)
+
+
+
security.ssl.internal.handshake-timeout
+
-1
+
Integer
+
The timeout (in ms) during SSL handshake. (-1 = use system default)
+
+
+
security.ssl.internal.session-cache-size
+
-1
+
Integer
+
The size of the cache used for storing SSL session objects. According to https://github.com/netty/netty/issues/832, you should always set this to an appropriate number to not run into a bug with stalling IO threads during garbage collection. (-1 = use system default).
+
+
+
security.ssl.internal.session-timeout
+
-1
+
Integer
+
The timeout (in ms) for the cached SSL session objects. (-1 = use system default)
+
+
+
security.ssl.provider
+
"JDK"
+
String
+
The SSL engine provider to use for the ssl transport:
`JDK`: default Java-based SSL engine
`OPENSSL`: openSSL-based SSL engine using system libraries
`OPENSSL` is based on netty-tcnative and comes in two flavours:
dynamically linked: This will use your system's openSSL libraries (if compatible) and requires `opt/flink-shaded-netty-tcnative-dynamic-*.jar` to be copied to `lib/`
statically linked: Due to potential licensing issues with openSSL (see LEGAL-393), we cannot ship pre-built libraries. However, you can build the required library yourself and put it into `lib/`: `git clone https://github.com/apache/flink-shaded.git && cd flink-shaded && mvn clean package -Pinclude-netty-tcnative-static -pl flink-shaded-netty-tcnative-static`
+
+
+
diff --git a/docs/_includes/generated/expert_state_backends_section.html b/docs/_includes/generated/expert_state_backends_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..9d50be1f4656225aa2b6fd5c475bde6108a2e0b0
--- /dev/null
+++ b/docs/_includes/generated/expert_state_backends_section.html
@@ -0,0 +1,30 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
state.backend.async
+
true
+
Boolean
+
Option whether the state backend should use an asynchronous snapshot method where possible and configurable. Some state backends may not support asynchronous snapshots, or only support asynchronous snapshots, and ignore this option.
+
+
+
state.backend.fs.memory-threshold
+
1024
+
Integer
+
The minimum size of state data files. All state chunks smaller than that are stored inline in the root checkpoint metadata file.
+
+
+
state.backend.fs.write-buffer-size
+
4096
+
Integer
+
The default size of the write buffer for the checkpoint streams that write to file systems. The actual write buffer size is determined to be the maximum of the value of this option and option 'state.backend.fs.memory-threshold'.
Delay between two consecutive restart attempts if `restart-strategy` has been set to `failure-rate`. It can be specified using notation: "1 min", "20 s"
The default filesystem scheme, used for paths that do not declare a scheme explicitly. May contain an authority, e.g. host:port in case of an HDFS NameNode.
fs.output.always-create-directory
false
+
Boolean
File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to "true", writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to "false", the writer will directly create the file directly at the output path, without creating a containing directory.
fs.overwrite-files
false
+
Boolean
Specifies whether file output writers should overwrite existing files by default. Set to "true" to overwrite by default,"false" otherwise.
The number of times that Flink retries the execution before the job is declared as failed if `restart-strategy` has been set to `fixed-delay`.
restart-strategy.fixed-delay.delay
-
"1 s"
+
1 s
+
Duration
Delay between two consecutive restart attempts if `restart-strategy` has been set to `fixed-delay`. Delaying the retries can be helpful when the program interacts with external systems where for example connections or pending transactions should reach a timeout before re-execution is attempted. It can be specified using notation: "1 min", "20 s"
Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER" or specify FQN of factory class.
high-availability.cluster-id
"/default"
+
String
The ID of the Flink cluster, used to separate multiple Flink clusters from each other. Needs to be set for standalone clusters but is automatically inferred in YARN and Mesos.
high-availability.jobmanager.port
"0"
-
Optional port (range) used by the job manager in high-availability mode.
+
String
+
The port (range) used by the Flink Master for its RPC connections in highly-available setups. In highly-available setups, this value is used instead of 'jobmanager.rpc.port'.A value of '0' means that a random free port is chosen. TaskManagers discover this port through the high-availability services (leader election), so a random port or a port range works without requiring any additional means of service discovery.
high-availability.storageDir
(none)
+
String
File system path (URI) where Flink persists metadata in high-availability setups.
+
+
high-availability.zookeeper.client.acl
+
"open"
+
String
+
Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).
Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).
Whether HistoryServer should cleanup jobs that are no longer present `historyserver.archive.fs.dir`.
+
historyserver.archive.fs.dir
(none)
+
String
Comma separated list of directories to fetch archived jobs from. The history server will monitor these directories for archived jobs. You can configure the JobManager to archive jobs to a directory via `jobmanager.archive.fs.dir`.
historyserver.archive.fs.refresh-interval
10000
+
Long
Interval in milliseconds for refreshing the archived job directories.
historyserver.web.address
(none)
+
String
Address of the HistoryServer's web interface.
historyserver.web.port
8082
+
Integer
Port of the HistoryServers's web interface.
historyserver.web.refresh-interval
10000
+
Long
The refresh interval for the HistoryServer web-frontend in milliseconds.
historyserver.web.ssl.enabled
false
+
Boolean
Enable HTTPs access to the HistoryServer web frontend. This is applicable only when the global SSL flag security.ssl.enabled is set to true.
historyserver.web.tmpdir
(none)
+
String
This configuration parameter allows defining the Flink web directory to be used by the history server web interface. The web interface will copy its static files into the directory.
diff --git a/docs/_includes/generated/influxdb_reporter_configuration.html b/docs/_includes/generated/influxdb_reporter_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..cb678f71c5ebe70e9ee316fce53c25120212e6bf
--- /dev/null
+++ b/docs/_includes/generated/influxdb_reporter_configuration.html
@@ -0,0 +1,66 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
connectTimeout
+
10000
+
Integer
+
(optional) the InfluxDB connect timeout for metrics
+
+
+
consistency
+
ONE
+
Enum
Possible values: [ALL, ANY, ONE, QUORUM]
+
(optional) the InfluxDB consistency level for metrics
+
+
+
db
+
(none)
+
String
+
the InfluxDB database to store metrics
+
+
+
host
+
(none)
+
String
+
the InfluxDB server host
+
+
+
password
+
(none)
+
String
+
(optional) InfluxDB username's password used for authentication
+
+
+
port
+
8086
+
Integer
+
the InfluxDB server port
+
+
+
retentionPolicy
+
(none)
+
String
+
(optional) the InfluxDB retention policy for metrics
+
+
+
username
+
(none)
+
String
+
(optional) InfluxDB username used for authentication
Dictionary for JobManager to store the archives of completed jobs.
jobmanager.execution.attempts-history-size
16
+
Integer
The maximum number of prior execution attempts kept in history.
jobmanager.execution.failover-strategy
-
"full"
+
region
+
String
This option specifies how the job computation recovers from task failures. Accepted values are:
'full': Restarts all tasks to recover the job.
'region': Restarts all tasks that could be affected by the task failure. More details can be found here.
jobmanager.heap.size
"1024m"
+
String
JVM heap size for the JobManager.
jobmanager.rpc.address
(none)
+
String
The config parameter defining the network address to connect to for communication with the job manager. This value is only interpreted in setups where a single JobManager with static name or address exists (simple standalone setups, or container setups with dynamic service name resolution). It is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby JobManagers.
jobmanager.rpc.port
6123
+
Integer
The config parameter defining the network port to connect to for communication with the job manager. Like jobmanager.rpc.address, this value is only interpreted in setups where a single JobManager with static name/address and port exists (simple standalone setups, or container setups with dynamic service name resolution). This config option is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby JobManagers.
jobstore.cache-size
52428800
+
Long
The job store cache size in bytes which is used to keep completed jobs in memory.
jobstore.expiration-time
3600
+
Long
The time in seconds after which a completed job expires and is purged from the job store.
jobstore.max-capacity
2147483647
+
Integer
The max number of completed jobs that can be kept in the job store.
slot.idle.timeout
50000
+
Long
The timeout in milliseconds for a idle slot in Slot Pool.
slot.request.timeout
300000
+
Long
The timeout in milliseconds for requesting a slot from Slot Pool.
A comma-separated list of login contexts to provide the Kerberos credentials to (for example, `Client,KafkaClient` to use the credentials for ZooKeeper authentication and for Kafka authentication)
security.kerberos.login.keytab
(none)
+
String
Absolute path to a Kerberos keytab file that contains the user credentials.
security.kerberos.login.principal
(none)
+
String
Kerberos principal name associated with the keytab.
security.kerberos.login.use-ticket-cache
true
+
Boolean
Indicates whether to read from your Kerberos ticket cache.
diff --git a/docs/_includes/generated/kubernetes_config_configuration.html b/docs/_includes/generated/kubernetes_config_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..86f8ec6b30d8102c9403301f9e161a2c835bca2c
--- /dev/null
+++ b/docs/_includes/generated/kubernetes_config_configuration.html
@@ -0,0 +1,108 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
kubernetes.cluster-id
+
(none)
+
String
+
The cluster id used for identifying the unique flink cluster. If it's not set, the client will generate a random UUID name.
+
+
+
kubernetes.config.file
+
(none)
+
String
+
The kubernetes config file will be used to create the client. The default is located at ~/.kube/config
Template for the kubernetes jobmanager and taskmanager container start invocation.
+
+
+
kubernetes.container.image
+
"flink:latest"
+
String
+
Image to use for Flink containers.
+
+
+
kubernetes.container.image.pull-policy
+
"IfNotPresent"
+
String
+
Kubernetes image pull policy. Valid values are Always, Never, and IfNotPresent. The default policy is IfNotPresent to avoid putting pressure to image repository.
+
+
+
kubernetes.container.image.pull-secrets
+
(none)
+
List<String>
+
A semicolon-separated list of the Kubernetes secrets used to access private image registries.
+
+
+
kubernetes.context
+
(none)
+
String
+
The desired context from your Kubernetes config file used to configure the Kubernetes client for interacting with the cluster. This could be helpful if one has multiple contexts configured and wants to administrate different Flink clusters on different Kubernetes clusters/contexts.
+
+
+
kubernetes.entry.path
+
"/opt/flink/bin/kubernetes-entry.sh"
+
String
+
The entrypoint script of kubernetes in the image. It will be used as command for jobmanager and taskmanager container.
+
+
+
kubernetes.flink.conf.dir
+
"/opt/flink/conf"
+
String
+
The flink conf directory that will be mounted in pod. The flink-conf.yaml, log4j.properties, logback.xml in this path will be overwritten from config map.
+
+
+
kubernetes.flink.log.dir
+
"/opt/flink/log"
+
String
+
The directory that logs of jobmanager and taskmanager be saved in the pod.
+
+
+
kubernetes.jobmanager.cpu
+
1.0
+
Double
+
The number of cpu used by job manager
+
+
+
kubernetes.jobmanager.service-account
+
"default"
+
String
+
Service account that is used by jobmanager within kubernetes cluster. The job manager uses this service account when requesting taskmanager pods from the API server.
+
+
+
kubernetes.namespace
+
"default"
+
String
+
The namespace that will be used for running the jobmanager and taskmanager pods.
+
+
+
kubernetes.rest-service.exposed.type
+
"LoadBalancer"
+
String
+
It could be ClusterIP/NodePort/LoadBalancer(default). When set to ClusterIP, the rest servicewill not be created.
+
+
+
kubernetes.service.create-timeout
+
"1 min"
+
String
+
Timeout used for creating the service. The timeout value requires a time-unit specifier (ms/s/min/h/d).
+
+
+
kubernetes.taskmanager.cpu
+
-1.0
+
Double
+
The number of cpu used by task manager. By default, the cpu is set to the number of slots per TaskManager
Amount of time to ask the Mesos master to not resend a declined resource offer again. This ensures a declined resource offer isn't resent immediately after being declined
mesos.resourcemanager.framework.name
"Flink"
+
String
Mesos framework name
mesos.resourcemanager.framework.principal
(none)
+
String
Mesos framework principal
mesos.resourcemanager.framework.role
"*"
+
String
Mesos framework role definition
mesos.resourcemanager.framework.secret
(none)
+
String
Mesos framework secret
mesos.resourcemanager.framework.user
(none)
+
String
Mesos framework user
mesos.resourcemanager.tasks.port-assignments
(none)
+
String
Comma-separated list of configuration keys which represent a configurable port. All port keys will dynamically get a port assigned through Mesos.
mesos.resourcemanager.unused-offer-expiration
120000
+
Long
Amount of time to wait for unused expired offers before declining them. This ensures your scheduler will not hoard unuseful offers.
Constraints for task placement on Mesos based on agent attributes. Takes a comma-separated list of key:value pairs corresponding to the attributes exposed by the target mesos agents. Example: az:eu-west-1a,series:t2
mesos.resourcemanager.tasks.bootstrap-cmd
(none)
+
String
A command which is executed before the TaskManager is started.
Custom parameters to be passed into docker run command when using the docker containerizer. Comma separated list of "key=value" pairs. The "value" may contain '='.
mesos.resourcemanager.tasks.container.image.name
(none)
+
String
Image name to use for the container.
mesos.resourcemanager.tasks.container.type
"mesos"
+
String
Type of the containerization used: “mesos” or “docker”.
mesos.resourcemanager.tasks.container.volumes
(none)
+
String
A comma separated list of [host_path:]container_path[:RO|RW]. This allows for mounting additional volumes into your container.
mesos.resourcemanager.tasks.cpus
0.0
+
Double
CPUs to assign to the Mesos workers.
mesos.resourcemanager.tasks.disk
0
+
Integer
Disk space to assign to the Mesos workers in MB.
mesos.resourcemanager.tasks.gpus
0
+
Integer
GPUs to assign to the Mesos workers.
mesos.resourcemanager.tasks.hostname
(none)
+
String
Optional value to define the TaskManager’s hostname. The pattern _TASK_ is replaced by the actual id of the Mesos task. This can be used to configure the TaskManager to use Mesos DNS (e.g. _TASK_.flink-service.mesos) for name lookups.
-
-
mesos.resourcemanager.tasks.mem
-
1024
-
Memory to assign to the Mesos workers in MB.
-
mesos.resourcemanager.tasks.taskmanager-cmd
"$FLINK_HOME/bin/mesos-taskmanager.sh"
+
String
mesos.resourcemanager.tasks.uris
(none)
+
String
A comma separated list of URIs of custom artifacts to be downloaded into the sandbox of Mesos workers.
taskmanager.numberOfTaskSlots
1
+
Integer
The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores).
Update interval for the metric fetcher used by the web UI in milliseconds. Decrease this value for faster updating metrics. Increase this value if the metric fetcher causes too much load. Setting this value to 0 disables the metric fetching completely.
metrics.internal.query-service.port
"0"
+
String
The port range used for Flink's internal metric query service. Accepts a list of ports (“50100,50101”), ranges(“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple Flink components are running on the same machine. Per default Flink will pick a random port.
metrics.internal.query-service.thread-priority
1
+
Integer
The thread priority used for Flink's internal metric query service. The thread is created by Akka's thread pool executor. The range of the priority is from 1 (MIN_PRIORITY) to 10 (MAX_PRIORITY). Warning, increasing this value may bring the main Flink components down.
metrics.latency.granularity
"operator"
+
String
Defines the granularity of latency metrics. Accepted values are:
single - Track latency without differentiating between sources and subtasks.
operator - Track latency while differentiating between sources, but not subtasks.
subtask - Track latency while differentiating between sources and subtasks.
metrics.latency.history-size
128
+
Integer
Defines the number of measured latencies to maintain at each operator.
metrics.latency.interval
0
+
Long
Defines the interval at which latency tracking marks are emitted from the sources. Disables latency tracking if set to 0 or a negative value. Enabling this feature can significantly impact the performance of the cluster.
metrics.reporter.<name>.<parameter>
(none)
+
String
Configures the parameter <parameter> for the reporter named <name>.
metrics.reporter.<name>.class
(none)
+
String
The reporter class to use for the reporter named <name>.
metrics.reporter.<name>.interval
(none)
+
String
The reporter interval to use for the reporter named <name>.
metrics.reporters
(none)
+
String
An optional list of reporter names. If configured, only reporters whose name matches any of the names in the list will be started. Otherwise, all reporters that could be found in the configuration will be started.
metrics.scope.delimiter
"."
+
String
Delimiter used to assemble the metric identifier.
metrics.scope.jm
"<host>.jobmanager"
+
String
Defines the scope format string that is applied to all metrics scoped to a JobManager.
metrics.scope.jm.job
"<host>.jobmanager.<job_name>"
+
String
Defines the scope format string that is applied to all metrics scoped to a job on a JobManager.
The task manager’s port used for data exchange operations.
taskmanager.data.ssl.enabled
true
+
Boolean
Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true
Boolean flag indicating whether the shuffle data will be compressed for blocking shuffle mode. Note that data is compressed per buffer and compression can incur extra CPU overhead, so it is more effective for IO bounded scenario when data compression ratio is high. Currently, shuffle data compression is an experimental feature and the config option can be changed in the future.
+
+
+
taskmanager.network.blocking-shuffle.type
+
"file"
+
String
+
The blocking shuffle type, either "mmap" or "file". The "auto" means selecting the property type automatically based on system memory architecture (64 bit for mmap and 32 bit for file). Note that the memory usage of mmap is not accounted by configured memory limits, but some resource frameworks like yarn would track this memory usage and kill the container once memory exceeding some threshold. Also note that this option is experimental and might be changed future.
+
taskmanager.network.detailed-metrics
false
+
Boolean
Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths.
taskmanager.network.memory.buffers-per-channel
2
+
Integer
Maximum number of network buffers to use for each outgoing/incoming channel (subpartition/input channel).In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization.
Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster.
The Netty send and receive buffer size. This defaults to the system buffer size (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.
+
+
+
taskmanager.network.netty.server.backlog
+
0
+
Integer
+
The netty server connection backlog.
+
+
+
taskmanager.network.netty.server.numThreads
+
-1
+
Integer
+
The number of Netty server threads.
+
+
+
taskmanager.network.netty.transport
+
"auto"
+
String
+
The Netty transport type, either "nio" or "epoll". The "auto" means selecting the property mode automatically based on the platform. Note that the "epoll" mode can get better performance, less GC and have more advanced features which are only available on modern Linux.
+
taskmanager.network.request-backoff.initial
100
+
Integer
Minimum backoff in milliseconds for partition requests of input channels.
taskmanager.network.request-backoff.max
10000
+
Integer
Maximum backoff in milliseconds for partition requests of input channels.
The Netty send and receive buffer size. This defaults to the system buffer size (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.
taskmanager.network.netty.server.backlog
0
+
Integer
The netty server connection backlog.
taskmanager.network.netty.server.numThreads
-1
+
Integer
The number of Netty server threads.
taskmanager.network.netty.transport
-
"nio"
-
The Netty transport type, either "nio" or "epoll"
+
"auto"
+
String
+
The Netty transport type, either "nio" or "epoll". The "auto" means selecting the property mode automatically based on the platform. Note that the "epoll" mode can get better performance, less GC and have more advanced features which are only available on modern Linux.
Strategy for aggregate phase. Only AUTO, TWO_PHASE or ONE_PHASE can be set.
AUTO: No special enforcer for aggregate stage. Whether to choose two stage aggregate or one stage aggregate depends on cost.
TWO_PHASE: Enforce to use two stage aggregate which has localAggregate and globalAggregate. Note that if aggregate call does not support optimize into two phase, we will still use one stage aggregate.
@@ -18,36 +20,43 @@ ONE_PHASE: Enforce to use one stage aggregate which only has CompleteGlobalAggre
table.optimizer.distinct-agg.split.bucket-num
Streaming
1024
+
Integer
Configure the number of buckets when splitting distinct aggregation. The number is used in the first level aggregation to calculate a bucket key 'hash_code(distinct_key) % BUCKET_NUM' which is used as an additional group key after splitting.
table.optimizer.distinct-agg.split.enabled
Streaming
false
+
Boolean
Tells the optimizer whether to split distinct aggregation (e.g. COUNT(DISTINCT col), SUM(DISTINCT col)) into two level. The first aggregation is shuffled by an additional key which is calculated using the hashcode of distinct_key and number of buckets. This optimization is very useful when there is data skew in distinct aggregation and gives the ability to scale-up the job. Default is false.
table.optimizer.join-reorder-enabled
BatchStreaming
false
+
Boolean
Enables join reorder in optimizer. Default is disabled.
table.optimizer.join.broadcast-threshold
Batch
1048576
+
Long
Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join. By setting this value to -1 to disable broadcasting.
table.optimizer.reuse-source-enabled
BatchStreaming
true
+
Boolean
When it is true, the optimizer will try to find out duplicated table sources and reuse them. This works only when table.optimizer.reuse-sub-plan-enabled is true.
table.optimizer.reuse-sub-plan-enabled
BatchStreaming
true
+
Boolean
When it is true, the optimizer will try to find out duplicated sub-plans and reuse them.
table.optimizer.source.predicate-pushdown-enabled
BatchStreaming
true
+
Boolean
When it is true, the optimizer will push down predicates into the FilterableTableSource. Default value is true.
he maximum number of line samples taken by the compiler for delimited inputs. The samples are used to estimate the number of records. This value can be overridden for a specific input with the input format’s parameters.
compiler.delimited-informat.max-sample-len
2097152
+
Integer
The maximal length of a line sample that the compiler takes for delimited inputs. If the length of a single sample exceeds this value (possible because of misconfiguration of the parser), the sampling aborts. This value can be overridden for a specific input with the input format’s parameters.
compiler.delimited-informat.min-line-samples
2
+
Integer
The minimum number of line samples taken by the compiler for delimited inputs. The samples are used to estimate the number of records. This value can be overridden for a specific input with the input format’s parameters
diff --git a/docs/_includes/generated/pipeline_configuration.html b/docs/_includes/generated/pipeline_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..f3fc216ddeaefa467e5e1968b8930096d3dbd2f9
--- /dev/null
+++ b/docs/_includes/generated/pipeline_configuration.html
@@ -0,0 +1,114 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
pipeline.auto-generate-uids
+
true
+
Boolean
+
When auto-generated UIDs are disabled, users are forced to manually specify UIDs on DataStream applications.
It is highly recommended that users specify UIDs before deploying to production since they are used to match state in savepoints to operators in a job. Because auto-generated ID's are likely to change when modifying a job, specifying custom IDs allow an application to evolve over time without discarding state.
+
+
+
pipeline.auto-type-registration
+
true
+
Boolean
+
Controls whether Flink is automatically registering all types in the user programs with Kryo.
+
+
+
pipeline.auto-watermark-interval
+
0 ms
+
Duration
+
The interval of the automatic watermark emission. Watermarks are used throughout the streaming system to keep track of the progress of time. They are used, for example, for time based windowing.
+
+
+
pipeline.cached-files
+
(none)
+
List<String>
+
Files to be registered at the distributed cache under the given name. The files will be accessible from any user-defined function in the (distributed) runtime under a local path. Files may be local files (which will be distributed via BlobServer), or files in a distributed file system. The runtime will copy the files temporarily to a local cache, if needed.
Forces Flink to use the Apache Avro serializer for POJOs.
Important: Make sure to include the `flink-avro` module.
+
+
+
pipeline.force-kryo
+
false
+
Boolean
+
If enabled, forces TypeExtractor to use Kryo serializer for POJOS even though we could analyze as POJO. In some cases this might be preferable. For example, when using interfaces with subclasses that cannot be analyzed as POJO.
+
+
+
pipeline.generic-types
+
true
+
Boolean
+
If the use of generic types is disabled, Flink will throw an `UnsupportedOperationException` whenever it encounters a data type that would go through Kryo for serialization.
Disabling generic types can be helpful to eagerly find and eliminate the use of types that would go through Kryo serialization during runtime. Rather than checking types individually, using this option will throw exceptions eagerly in the places where generic types are used.
We recommend to use this option only during development and pre-production phases, not during actual production use. The application program and/or the input data may be such that new, previously unseen, types occur at some point. In that case, setting this option would cause the program to fail.
+
+
+
pipeline.global-job-parameters
+
(none)
+
Map
+
Register a custom, serializable user configuration object. The configuration can be accessed in operators
+
+
+
pipeline.jars
+
(none)
+
List<String>
+
A semicolon-separated list of the jars to package with the job jars to be sent to the cluster. These have to be valid paths.
+
+
+
pipeline.max-parallelism
+
-1
+
Integer
+
The program-wide maximum parallelism used for operators which haven't specified a maximum parallelism. The maximum parallelism specifies the upper limit for dynamic scaling and the number of key groups used for partitioned state.
+
+
+
pipeline.object-reuse
+
false
+
Boolean
+
When enabled objects that Flink internally uses for deserialization and passing data to user-code functions will be reused. Keep in mind that this can lead to bugs when the user-code function of an operation is not aware of this behaviour.
+
+
+
pipeline.operator-chaining
+
true
+
Boolean
+
Operator chaining allows non-shuffle operations to be co-located in the same thread fully avoiding serialization and de-serialization.
+
+
+
pipeline.registered-kryo-types
+
(none)
+
List<String>
+
Semicolon separated list of types to be registered with the serialization stack. If the type is eventually serialized as a POJO, then the type is registered with the POJO serializer. If the type ends up being serialized with Kryo, then it will be registered at Kryo to make sure that only tags are written.
+
+
+
pipeline.registered-pojo-types
+
(none)
+
List<String>
+
Semicolon separated list of types to be registered with the serialization stack. If the type is eventually serialized as a POJO, then the type is registered with the POJO serializer. If the type ends up being serialized with Kryo, then it will be registered at Kryo to make sure that only tags are written.
Specifies whether to delete metrics from the PushGateway on shutdown.
filterLabelValueCharacters
true
+
Boolean
Specifies whether to filter label value characters. If enabled, all characters not matching [a-zA-Z0-9:_] will be removed, otherwise no characters will be removed. Before disabling this option please ensure that your label values meet the Prometheus requirements.
groupingKey
(none)
+
String
Specifies the grouping key which is the group and global labels of all metrics. The label name and value are separated by '=', and labels are separated by ';', e.g., `k1=v1;k2=v2`. Please ensure that your grouping key meets the Prometheus requirements.
host
(none)
+
String
The PushGateway server host.
jobName
(none)
+
String
The job name under which metrics will be pushed
port
-1
+
Integer
The PushGateway server port.
randomJobNameSuffix
true
+
Boolean
Specifies whether a random suffix should be appended to the job name.
diff --git a/docs/_includes/generated/python_configuration.html b/docs/_includes/generated/python_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..212cec28805c70d330f757b74008426fea7ca7f0
--- /dev/null
+++ b/docs/_includes/generated/python_configuration.html
@@ -0,0 +1,42 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
python.fn-execution.arrow.batch.size
+
1000
+
Integer
+
The maximum number of elements to include in an arrow batch for Python user-defined function execution. The arrow batch size should not exceed the bundle size. Otherwise, the bundle size will be used as the arrow batch size.
+
+
+
python.fn-execution.buffer.memory.size
+
"15mb"
+
String
+
The amount of memory to be allocated by the input buffer and output buffer of a Python worker. The memory will be accounted as managed memory if the actual memory allocated to an operator is no less than the total memory of a Python worker. Otherwise, this configuration takes no effect.
+
+
+
python.fn-execution.bundle.size
+
1000
+
Integer
+
The maximum number of elements to include in a bundle for Python user-defined function execution. The elements are processed asynchronously. One bundle of elements are processed before processing the next bundle of elements. A larger value can improve the throughput, but at the cost of more memory usage and higher latency.
+
+
+
python.fn-execution.bundle.time
+
1000
+
Long
+
Sets the waiting timeout(in milliseconds) before processing a bundle for Python user-defined function execution. The timeout defines how long the elements of a bundle will be buffered before being processed. Lower timeouts lead to lower tail latencies, but may affect throughput.
+
+
+
python.fn-execution.framework.memory.size
+
"64mb"
+
String
+
The amount of memory to be allocated by the Python framework. The sum of the value of this configuration and "python.fn-execution.buffer.memory.size" represents the total memory of a Python worker. The memory will be accounted as managed memory if the actual memory allocated to an operator is no less than the total memory of a Python worker. Otherwise, this configuration takes no effect.
Number of network (Netty's event loop) Threads for queryable state client.
queryable-state.enable
false
+
Boolean
Option whether the queryable state proxy and server should be enabled where possible and configurable.
queryable-state.proxy.network-threads
0
+
Integer
Number of network (Netty's event loop) Threads for queryable state proxy.
queryable-state.proxy.ports
"9069"
+
String
The port range of the queryable state proxy. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234".
queryable-state.proxy.query-threads
0
+
Integer
Number of query Threads for queryable state proxy. Uses the number of slots if set to 0.
queryable-state.server.network-threads
0
+
Integer
Number of network (Netty's event loop) Threads for queryable state server.
queryable-state.server.ports
"9067"
+
String
The port range of the queryable state server. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234".
queryable-state.server.query-threads
0
+
Integer
Number of query Threads for queryable state server. Uses the number of slots if set to 0.
Minimum amount of heap memory to remove in containers, as a safety margin.
+
Integer
+
Minimum amount of heap memory to remove in Job Master containers, as a safety margin.
containerized.heap-cutoff-ratio
0.25
-
Percentage of heap space to remove from containers (YARN / Mesos), to compensate for other JVM memory usage.
-
-
-
local.number-resourcemanager
-
1
-
The number of resource managers start.
+
Float
+
Percentage of heap space to remove from Job Master containers (YARN / Mesos / Kubernetes), to compensate for other JVM memory usage.
resourcemanager.job.timeout
"5 minutes"
+
String
Timeout for jobs which don't have a job manager as leader assigned.
resourcemanager.rpc.port
0
+
Integer
Defines the network port to connect to for communication with the resource manager. By default, the port of the JobManager, because the same ActorSystem is used. Its not possible to use this configuration key to define port ranges.
resourcemanager.standalone.start-up-time
-1
+
Long
Time in milliseconds of the start-up period of a standalone cluster. During this time, resource manager of the standalone cluster expects new task executors to be registered, and will not fail slot requests that can not be satisfied by any current registered slots. After this time, it will fail pending and new coming requests immediately that can not be satisfied by registered slots. If not set, 'slotmanager.request-timeout' will be used by default.
resourcemanager.taskmanager-timeout
30000
+
Long
The timeout for an idle task manager to be released.
The address that should be used by clients to connect to the server.
rest.await-leader-timeout
30000
+
Long
The time in ms that the client waits for the leader address, e.g., Dispatcher or WebMonitorEndpoint
rest.bind-address
(none)
+
String
The address that the server binds itself.
rest.bind-port
"8081"
+
String
The port that the server binds itself. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple Rest servers are running on the same machine.
rest.client.max-content-length
104857600
+
Integer
The maximum content length in bytes that the client will handle.
rest.connection-timeout
15000
+
Long
The maximum time in ms for the client to establish a TCP connection.
rest.idleness-timeout
300000
+
Long
The maximum time in ms for a connection to stay idle before failing.
rest.port
8081
+
Integer
The port that the client connects to. If rest.bind-port has not been specified, then the REST server will bind to this port.
rest.retry.delay
3000
+
Long
The time in ms that the client waits between retries (See also `rest.retry.max-attempts`).
rest.retry.max-attempts
20
+
Integer
The number of retries the client will attempt if a retryable operations fails.
rest.server.max-content-length
104857600
+
Integer
The maximum content length in bytes that the server will handle.
rest.server.numThreads
4
+
Integer
The number of threads for the asynchronous processing of requests.
rest.server.thread-priority
5
+
Integer
Thread priority of the REST server's executor for processing asynchronous requests. Lowering the thread priority will give Flink's main components more CPU time whereas increasing will allocate more time for the REST server's processing.
Defines the restart strategy to use in case of job failures. Accepted values are:
`none`, `off`, `disable`: No restart strategy.
`fixeddelay`, `fixed-delay`: Fixed delay restart strategy. More details can be found here.
`failurerate`, `failure-rate`: Failure rate restart strategy. More details can be found here.
If checkpointing is disabled, the default value is `none`. If checkpointing is enabled, the default value is `fixed-delay` with `Integer.MAX_VALUE` restart attempts and '`1 s`' delay.
If true, RocksDB will pick target size of each level dynamically. From an empty DB, RocksDB would make last level the base level, which means merging L0 data into the last level, until it exceeds max_bytes_for_level_base. And then repeat this process for second last level and so on. RocksDB has default configuration as 'false'. For more information, please refer to RocksDB's doc.
state.backend.rocksdb.compaction.style
(none)
+
Enum
Possible values: [LEVEL, UNIVERSAL, FIFO]
The specified compaction style for DB. Candidate compaction style is LEVEL, FIFO or UNIVERSAL, and RocksDB choose 'LEVEL' as default style.
state.backend.rocksdb.files.open
(none)
-
The maximum number of open files that can be used by the DB, '-1' means no limit. RocksDB has default configuration as '5000'.
+
Integer
+
The maximum number of open files (per TaskManager) that can be used by the DB, '-1' means no limit. RocksDB has default configuration as '-1'.
state.backend.rocksdb.thread.num
(none)
-
The maximum number of concurrent background flush and compaction jobs. RocksDB has default configuration as '1'.
+
Integer
+
The maximum number of concurrent background flush and compaction jobs (per TaskManager). RocksDB has default configuration as '1'.
+
+
+
state.backend.rocksdb.write-batch-size
+
2 mb
+
MemorySize
+
The max size of the consumed memory for RocksDB batch write, will flush just based on item count if this config set to 0.
state.backend.rocksdb.writebuffer.count
(none)
+
Integer
Tne maximum number of write buffers that are built up in memory. RocksDB has default configuration as '2'.
state.backend.rocksdb.writebuffer.number-to-merge
(none)
+
Integer
The minimum number of write buffers that will be merged together before writing to storage. RocksDB has default configuration as '1'.
state.backend.rocksdb.writebuffer.size
(none)
-
The amount of data built up in memory (backed by an unsorted log on disk) before converting to a sorted on-disk files. RocksDB has default writebuffer size as '4MB'.
+
MemorySize
+
The amount of data built up in memory (backed by an unsorted log on disk) before converting to a sorted on-disk files. RocksDB has default writebuffer size as '64MB'.
The number of threads used to transfer (download and upload) files in RocksDBStateBackend.
+
Integer
+
The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.
state.backend.rocksdb.localdir
(none)
+
String
The local directory (on the TaskManager) where RocksDB puts its files.
+
+
state.backend.rocksdb.memory.fixed-per-slot
+
(none)
+
MemorySize
+
The fixed total amount of memory, shared among all RocksDB instances per slot. This option overrides the 'state.backend.rocksdb.memory.managed' option when configured. If neither this option, nor the 'state.backend.rocksdb.memory.managed' optionare set, then each RocksDB column family state has its own memory caches (as controlled by the column family options).
+
+
+
state.backend.rocksdb.memory.high-prio-pool-ratio
+
0.1
+
Double
+
The fraction of cache memory that is reserved for high-priority data like index, filter, and compression dictionary blocks. This option only has an effect when 'state.backend.rocksdb.memory.managed' or 'state.backend.rocksdb.memory.fixed-per-slot' are configured.
+
+
+
state.backend.rocksdb.memory.managed
+
true
+
Boolean
+
If set, the RocksDB state backend will automatically configure itself to use the managed memory budget of the task slot, and divide the memory over write buffers, indexes, block caches, etc. That way, the three major uses of memory of RocksDB will be capped.
+
+
+
state.backend.rocksdb.memory.write-buffer-ratio
+
0.5
+
Double
+
The maximum amount of memory that write buffers may take, as a fraction of the total shared memory. This option only has an effect when 'state.backend.rocksdb.memory.managed' or 'state.backend.rocksdb.memory.fixed-per-slot' are configured.
The options factory class for RocksDB to create DBOptions and ColumnFamilyOptions. The default options factory is org.apache.flink.contrib.streaming.state.DefaultConfigurableOptionsFactory, and it would read the configured options which provided in 'RocksDBConfigurableOptions'.
state.backend.rocksdb.predefined-options
"DEFAULT"
+
String
The predefined settings for RocksDB DBOptions and ColumnFamilyOptions by Flink community. Current supported candidate predefined-options are DEFAULT, SPINNING_DISK_OPTIMIZED, SPINNING_DISK_OPTIMIZED_HIGH_MEM or FLASH_SSD_OPTIMIZED. Note that user customized options and options from the OptionsFactory are applied on top of these predefined ones.
state.backend.rocksdb.timer-service.factory
-
"HEAP"
+
"ROCKSDB"
+
String
This determines the factory for timer service state implementation. Options are either HEAP (heap-based, default) or ROCKSDB for an implementation based on RocksDB .
This determines if compaction filter to cleanup state with TTL is enabled for backend.Note: User can still decide in state TTL configuration in state descriptor whether the filter is active for particular state or not.
Estimated total number of bytes compaction needs to rewrite to get all levels down to under target size. Not valid for other compactions than level-based.
Monitor the number of immutable memtables in RocksDB.
state.backend.rocksdb.metrics.num-live-versions
false
+
Boolean
Monitor number of live versions. Version is an internal data structure. See RocksDB file version_set.h for details. More live versions often mean more SST files are held from being deleted, by iterators or unfinished compactions.
Monitor the total size (bytes) of all SST files.WARNING: may slow down online queries if there are too many files.
diff --git a/docs/_includes/generated/savepoint_config_configuration.html b/docs/_includes/generated/savepoint_config_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..2f9b464e593cb315baa3ca689b3de3854df60098
--- /dev/null
+++ b/docs/_includes/generated/savepoint_config_configuration.html
@@ -0,0 +1,24 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
execution.savepoint.ignore-unclaimed-state
+
false
+
Boolean
+
Allow to skip savepoint state that cannot be restored. Allow this if you removed an operator from your pipeline after the savepoint was triggered.
+
+
+
execution.savepoint.path
+
(none)
+
String
+
Path to a savepoint to restore the job from (for example hdfs:///flink/savepoint-1537).
+
+
+
diff --git a/docs/_includes/generated/security_auth_kerberos_section.html b/docs/_includes/generated/security_auth_kerberos_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..cf279171a54c9317a7cf86c064d7010062c54b49
--- /dev/null
+++ b/docs/_includes/generated/security_auth_kerberos_section.html
@@ -0,0 +1,36 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
security.kerberos.login.contexts
+
(none)
+
String
+
A comma-separated list of login contexts to provide the Kerberos credentials to (for example, `Client,KafkaClient` to use the credentials for ZooKeeper authentication and for Kafka authentication)
+
+
+
security.kerberos.login.keytab
+
(none)
+
String
+
Absolute path to a Kerberos keytab file that contains the user credentials.
+
+
+
security.kerberos.login.principal
+
(none)
+
String
+
Kerberos principal name associated with the keytab.
+
+
+
security.kerberos.login.use-ticket-cache
+
true
+
Boolean
+
Indicates whether to read from your Kerberos ticket cache.
+
+
+
diff --git a/docs/_includes/generated/security_auth_zk_section.html b/docs/_includes/generated/security_auth_zk_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..7aaaaff20d78b8bbe3bcacf9cc01c3f0fa2997f1
--- /dev/null
+++ b/docs/_includes/generated/security_auth_zk_section.html
@@ -0,0 +1,30 @@
+
List of factories that should be used to instantiate a security context. If multiple are configured, Flink will use the first compatible factory. You should have a NoOpSecurityContextFactory in this list as a fallback.
+
+
+
security.kerberos.login.contexts
+
(none)
+
String
+
A comma-separated list of login contexts to provide the Kerberos credentials to (for example, `Client,KafkaClient` to use the credentials for ZooKeeper authentication and for Kafka authentication)
+
+
+
security.kerberos.login.keytab
+
(none)
+
String
+
Absolute path to a Kerberos keytab file that contains the user credentials.
+
+
+
security.kerberos.login.principal
+
(none)
+
String
+
Kerberos principal name associated with the keytab.
+
+
+
security.kerberos.login.use-ticket-cache
+
true
+
Boolean
+
Indicates whether to read from your Kerberos ticket cache.
List of factories that should be used to instantiate security modules. All listed modules will be installed. Keep in mind that the configured security context might rely on some modules being present.
+
security.ssl.algorithms
"TLS_RSA_WITH_AES_128_CBC_SHA"
+
String
The comma separated list of standard SSL algorithms to be supported. Read more here
+
+
security.ssl.internal.cert.fingerprint
+
(none)
+
String
+
The sha1 fingerprint of the internal certificate. This further protects the internal communication to present the exact certificate used by Flink.This is necessary where one cannot use private CA(self signed) or there is internal firm wide CA is required
+
security.ssl.internal.close-notify-flush-timeout
-1
+
Integer
The timeout (in ms) for flushing the `close_notify` that was triggered by closing a channel. If the `close_notify` was not flushed in the given timeout the channel will be closed forcibly. (-1 = use system default)
security.ssl.internal.enabled
false
+
Boolean
Turns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc).
security.ssl.internal.handshake-timeout
-1
+
Integer
The timeout (in ms) during SSL handshake. (-1 = use system default)
security.ssl.internal.key-password
(none)
+
String
The secret to decrypt the key in the keystore for Flink's internal endpoints (rpc, data transport, blob server).
security.ssl.internal.keystore
(none)
+
String
The Java keystore file with SSL Key and Certificate, to be used Flink's internal endpoints (rpc, data transport, blob server).
security.ssl.internal.keystore-password
(none)
+
String
The secret to decrypt the keystore file for Flink's for Flink's internal endpoints (rpc, data transport, blob server).
security.ssl.internal.session-cache-size
-1
+
Integer
The size of the cache used for storing SSL session objects. According to https://github.com/netty/netty/issues/832, you should always set this to an appropriate number to not run into a bug with stalling IO threads during garbage collection. (-1 = use system default).
security.ssl.internal.session-timeout
-1
+
Integer
The timeout (in ms) for the cached SSL session objects. (-1 = use system default)
security.ssl.internal.truststore
(none)
+
String
The truststore file containing the public CA certificates to verify the peer for Flink's internal endpoints (rpc, data transport, blob server).
security.ssl.internal.truststore-password
(none)
+
String
The password to decrypt the truststore for Flink's internal endpoints (rpc, data transport, blob server).
-
-
security.ssl.key-password
-
(none)
-
The secret to decrypt the server key in the keystore.
-
-
-
security.ssl.keystore
-
(none)
-
The Java keystore file to be used by the flink endpoint for its SSL Key and Certificate.
-
-
-
security.ssl.keystore-password
-
(none)
-
The secret to decrypt the keystore file.
-
security.ssl.protocol
"TLSv1.2"
+
String
The SSL protocol version to be supported for the ssl transport. Note that it doesn’t support comma separated list.
security.ssl.provider
"JDK"
+
String
The SSL engine provider to use for the ssl transport:
`JDK`: default Java-based SSL engine
`OPENSSL`: openSSL-based SSL engine using system libraries
`OPENSSL` is based on netty-tcnative and comes in two flavours:
dynamically linked: This will use your system's openSSL libraries (if compatible) and requires `opt/flink-shaded-netty-tcnative-dynamic-*.jar` to be copied to `lib/`
statically linked: Due to potential licensing issues with openSSL (see LEGAL-393), we cannot ship pre-built libraries. However, you can build the required library yourself and put it into `lib/`: `git clone https://github.com/apache/flink-shaded.git && cd flink-shaded && mvn clean package -Pinclude-netty-tcnative-static -pl flink-shaded-netty-tcnative-static`
security.ssl.rest.authentication-enabled
false
+
Boolean
Turns on mutual SSL authentication for external communication via the REST endpoints.
+
+
security.ssl.rest.cert.fingerprint
+
(none)
+
String
+
The sha1 fingerprint of the rest certificate. This further protects the rest REST endpoints to present certificate which is only used by proxy serverThis is necessary where once uses public CA or internal firm wide CA
+
security.ssl.rest.enabled
false
+
Boolean
Turns on SSL for external communication via the REST endpoints.
security.ssl.rest.key-password
(none)
+
String
The secret to decrypt the key in the keystore for Flink's external REST endpoints.
security.ssl.rest.keystore
(none)
+
String
The Java keystore file with SSL Key and Certificate, to be used Flink's external REST endpoints.
security.ssl.rest.keystore-password
(none)
+
String
The secret to decrypt the keystore file for Flink's for Flink's external REST endpoints.
security.ssl.rest.truststore
(none)
+
String
The truststore file containing the public CA certificates to verify the peer for Flink's external REST endpoints.
security.ssl.rest.truststore-password
(none)
+
String
The password to decrypt the truststore for Flink's external REST endpoints.
-
security.ssl.truststore
-
(none)
-
The truststore file containing the public CA certificates to be used by flink endpoints to verify the peer’s certificate.
+
security.ssl.verify-hostname
+
true
+
Boolean
+
Flag to enable peer’s hostname verification during ssl handshake.
-
security.ssl.truststore-password
-
(none)
-
The secret to decrypt the truststore.
+
zookeeper.sasl.disable
+
false
+
Boolean
+
-
security.ssl.verify-hostname
-
true
-
Flag to enable peer’s hostname verification during ssl handshake.
+
zookeeper.sasl.login-context-name
+
"Client"
+
String
+
+
+
+
zookeeper.sasl.service-name
+
"zookeeper"
+
String
+
diff --git a/docs/_includes/generated/security_ssl_section.html b/docs/_includes/generated/security_ssl_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..ae345d8e4c320bcf592d4a66895b90d35cf51e94
--- /dev/null
+++ b/docs/_includes/generated/security_ssl_section.html
@@ -0,0 +1,120 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
security.ssl.algorithms
+
"TLS_RSA_WITH_AES_128_CBC_SHA"
+
String
+
The comma separated list of standard SSL algorithms to be supported. Read more here
+
+
+
security.ssl.internal.cert.fingerprint
+
(none)
+
String
+
The sha1 fingerprint of the internal certificate. This further protects the internal communication to present the exact certificate used by Flink.This is necessary where one cannot use private CA(self signed) or there is internal firm wide CA is required
+
+
+
security.ssl.internal.enabled
+
false
+
Boolean
+
Turns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc).
+
+
+
security.ssl.internal.key-password
+
(none)
+
String
+
The secret to decrypt the key in the keystore for Flink's internal endpoints (rpc, data transport, blob server).
+
+
+
security.ssl.internal.keystore
+
(none)
+
String
+
The Java keystore file with SSL Key and Certificate, to be used Flink's internal endpoints (rpc, data transport, blob server).
+
+
+
security.ssl.internal.keystore-password
+
(none)
+
String
+
The secret to decrypt the keystore file for Flink's for Flink's internal endpoints (rpc, data transport, blob server).
+
+
+
security.ssl.internal.truststore
+
(none)
+
String
+
The truststore file containing the public CA certificates to verify the peer for Flink's internal endpoints (rpc, data transport, blob server).
+
+
+
security.ssl.internal.truststore-password
+
(none)
+
String
+
The password to decrypt the truststore for Flink's internal endpoints (rpc, data transport, blob server).
+
+
+
security.ssl.protocol
+
"TLSv1.2"
+
String
+
The SSL protocol version to be supported for the ssl transport. Note that it doesn’t support comma separated list.
+
+
+
security.ssl.rest.authentication-enabled
+
false
+
Boolean
+
Turns on mutual SSL authentication for external communication via the REST endpoints.
+
+
+
security.ssl.rest.cert.fingerprint
+
(none)
+
String
+
The sha1 fingerprint of the rest certificate. This further protects the rest REST endpoints to present certificate which is only used by proxy serverThis is necessary where once uses public CA or internal firm wide CA
+
+
+
security.ssl.rest.enabled
+
false
+
Boolean
+
Turns on SSL for external communication via the REST endpoints.
+
+
+
security.ssl.rest.key-password
+
(none)
+
String
+
The secret to decrypt the key in the keystore for Flink's external REST endpoints.
+
+
+
security.ssl.rest.keystore
+
(none)
+
String
+
The Java keystore file with SSL Key and Certificate, to be used Flink's external REST endpoints.
+
+
+
security.ssl.rest.keystore-password
+
(none)
+
String
+
The secret to decrypt the keystore file for Flink's for Flink's external REST endpoints.
+
+
+
security.ssl.rest.truststore
+
(none)
+
String
+
The truststore file containing the public CA certificates to verify the peer for Flink's external REST endpoints.
+
+
+
security.ssl.rest.truststore-password
+
(none)
+
String
+
The password to decrypt the truststore for Flink's external REST endpoints.
+
+
+
security.ssl.verify-hostname
+
true
+
Boolean
+
Flag to enable peer’s hostname verification during ssl handshake.
The full class name of the shuffle service factory implementation to be used by the cluster. The default implementation uses Netty for network communication and local memory as well disk space to store results on a TaskExecutor.
diff --git a/docs/_includes/generated/state_backend_rocksdb_section.html b/docs/_includes/generated/state_backend_rocksdb_section.html
new file mode 100644
index 0000000000000000000000000000000000000000..974c5c12abed3cf7765432929a13f384f208655a
--- /dev/null
+++ b/docs/_includes/generated/state_backend_rocksdb_section.html
@@ -0,0 +1,42 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
state.backend.rocksdb.memory.fixed-per-slot
+
(none)
+
MemorySize
+
The fixed total amount of memory, shared among all RocksDB instances per slot. This option overrides the 'state.backend.rocksdb.memory.managed' option when configured. If neither this option, nor the 'state.backend.rocksdb.memory.managed' optionare set, then each RocksDB column family state has its own memory caches (as controlled by the column family options).
+
+
+
state.backend.rocksdb.memory.high-prio-pool-ratio
+
0.1
+
Double
+
The fraction of cache memory that is reserved for high-priority data like index, filter, and compression dictionary blocks. This option only has an effect when 'state.backend.rocksdb.memory.managed' or 'state.backend.rocksdb.memory.fixed-per-slot' are configured.
+
+
+
state.backend.rocksdb.memory.managed
+
true
+
Boolean
+
If set, the RocksDB state backend will automatically configure itself to use the managed memory budget of the task slot, and divide the memory over write buffers, indexes, block caches, etc. That way, the three major uses of memory of RocksDB will be capped.
+
+
+
state.backend.rocksdb.memory.write-buffer-ratio
+
0.5
+
Double
+
The maximum amount of memory that write buffers may take, as a fraction of the total shared memory. This option only has an effect when 'state.backend.rocksdb.memory.managed' or 'state.backend.rocksdb.memory.fixed-per-slot' are configured.
+
+
+
state.backend.rocksdb.timer-service.factory
+
"ROCKSDB"
+
String
+
This determines the factory for timer service state implementation. Options are either HEAP (heap-based, default) or ROCKSDB for an implementation based on RocksDB .
+
+
+
diff --git a/docs/_includes/generated/stream_pipeline_configuration.html b/docs/_includes/generated/stream_pipeline_configuration.html
new file mode 100644
index 0000000000000000000000000000000000000000..0fdefe9c4f17a06129e936232d909e3ee1ca5fe0
--- /dev/null
+++ b/docs/_includes/generated/stream_pipeline_configuration.html
@@ -0,0 +1,18 @@
+
+
+
+
Key
+
Default
+
Type
+
Description
+
+
+
+
+
pipeline.time-characteristic
+
ProcessingTime
+
Enum
Possible values: [ProcessingTime, IngestionTime, EventTime]
+
The time characteristic for all created streams, e.g., processingtime, event time, or ingestion time.
If you set the characteristic to IngestionTime or EventTime this will set a default watermark update interval of 200 ms. If this is not applicable for your application you should change it using `pipeline.auto-watermark-interval`.
Time interval between two successive task cancellation attempts in milliseconds.
task.cancellation.timeout
180000
+
Long
Timeout in milliseconds after which a task cancellation times out and leads to a fatal TaskManager error. A value of 0 deactivates the watch dog.
task.cancellation.timers.timeout
7500
+
Long
Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.
-
-
task.checkpoint.alignment.max-size
-
-1
-
The maximum number of bytes that a checkpoint alignment may buffer. If the checkpoint alignment buffers more than the configured amount of data, the checkpoint is aborted (skipped). A value of -1 indicates that there is no limit.
-
taskmanager.debug.memory.log
false
+
Boolean
Flag indicating whether to start a thread, which repeatedly logs the memory usage of the JVM.
taskmanager.debug.memory.log-interval
5000
+
Long
The interval (in ms) for the log thread to log the current memory usage.
-
-
taskmanager.exit-on-fatal-akka-error
-
false
-
Whether the quarantine monitor for task managers shall be started. The quarantine monitor shuts down the actor system if it detects that it has quarantined another actor system or if it has been quarantined by another actor system.
-
taskmanager.host
(none)
+
String
The address of the network interface that the TaskManager binds to. This option can be used to define explicitly a binding address. Because different TaskManagers need different values for this option, usually it is specified in an additional non-shared TaskManager-specific config file.
taskmanager.jvm-exit-on-oom
false
+
Boolean
Whether to kill the TaskManager when the task thread throws an OutOfMemoryError.
taskmanager.network.bind-policy
"ip"
+
String
The automatic address binding policy used by the TaskManager if "taskmanager.host" is not set. The value should be one of the following:
"name" - uses hostname as binding address
"ip" - uses host's ip address as binding address
taskmanager.numberOfTaskSlots
1
+
Integer
The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores).
taskmanager.registration.initial-backoff
-
"500 ms"
+
500 ms
+
Duration
The initial registration backoff between two consecutive registration attempts. The backoff is doubled for each new registration attempt until it reaches the maximum registration backoff.
taskmanager.registration.max-backoff
-
"30 s"
+
30 s
+
Duration
The maximum registration backoff between two consecutive registration attempts. The max registration backoff requires a time unit specifier (ms/s/min/h/d).
taskmanager.registration.refused-backoff
-
"10 s"
+
10 s
+
Duration
The backoff after a registration has been refused by the job manager before retrying to connect.
taskmanager.registration.timeout
-
"5 min"
+
5 min
+
Duration
Defines the timeout for the TaskManager registration. If the duration is exceeded without a successful registration, then the TaskManager terminates.
taskmanager.rpc.port
"0"
+
String
The task manager’s IPC port. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple TaskManagers are running on the same machine.
Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Network Memory. See also 'taskmanager.memory.process.size' for total process memory size configuration.
+
taskmanager.memory.framework.heap.size
-
"128m"
+
128 mb
+
MemorySize
Framework Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for TaskExecutor framework, which will not be allocated to task slots.
+
+
taskmanager.memory.framework.off-heap.size
+
128 mb
+
MemorySize
+
Framework Off-Heap Memory size for TaskExecutors. This is the size of off-heap memory (JVM direct memory and native memory) reserved for TaskExecutor framework, which will not be allocated to task slots. The configured value will be fully counted when Flink calculates the JVM max direct memory size parameter.
+
taskmanager.memory.jvm-metaspace.size
-
"192m"
+
96 mb
+
MemorySize
JVM Metaspace Size for the TaskExecutors.
taskmanager.memory.jvm-overhead.fraction
0.1
-
Fraction of Total Process Memory to be reserved for JVM Overhead. This is off-heap memory reserved for JVM overhead, such as thread stack space, I/O direct memory, compile cache, etc. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
+
Float
+
Fraction of Total Process Memory to be reserved for JVM Overhead. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
taskmanager.memory.jvm-overhead.max
-
"1g"
-
Max JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, I/O direct memory, compile cache, etc. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
+
1 gb
+
MemorySize
+
Max JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
taskmanager.memory.jvm-overhead.min
-
"128m"
-
Min JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, I/O direct memory, compile cache, etc. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
+
192 mb
+
MemorySize
+
Min JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value.
taskmanager.memory.managed.fraction
-
0.5
+
0.4
+
Float
Fraction of Total Flink Memory to be used as Managed Memory, if Managed Memory size is not explicitly specified.
-
-
taskmanager.memory.managed.off-heap.fraction
-
-1.0
-
Fraction of Managed Memory that Off-Heap Managed Memory takes, if Off-Heap Managed Memory size is not explicitly specified. If the fraction is not explicitly specified (or configured with negative values), it will be derived from the legacy config option 'taskmanager.memory.off-heap', to use either all on-heap memory or all off-heap memory for Managed Memory.
-
-
-
taskmanager.memory.managed.off-heap.size
-
(none)
-
Off-Heap Managed Memory size for TaskExecutors. This is the part of Managed Memory that is off-heap, while the remaining is on-heap. If unspecified, it will be derived to make up the configured fraction of the Managed Memory size.
-
taskmanager.memory.managed.size
(none)
-
Managed Memory size for TaskExecutors. This is the size of memory managed by the memory manager, including both On-Heap Managed Memory and Off-Heap Managed Memory, reserved for sorting, hash tables, caching of intermediate results and state backends. Memory consumers can either allocate memory from the memory manager in the form of MemorySegments, or reserve bytes from the memory manager and keep their memory usage within that boundary. If unspecified, it will be derived to make up the configured fraction of the Total Flink Memory.
-
-
-
taskmanager.memory.preallocate
-
false
-
Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting. When `taskmanager.memory.off-heap` is set to true, then it is advised that this configuration is also set to true. If this configuration is set to false cleaning up of the allocated off-heap memory happens only when the configured JVM parameter MaxDirectMemorySize is reached by triggering a full GC. For streaming setups, it is highly recommended to set this value to false as the core state backends currently do not use the managed memory.
+
MemorySize
+
Managed Memory size for TaskExecutors. This is the size of off-heap memory managed by the memory manager, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backend. Memory consumers can either allocate memory from the memory manager in the form of MemorySegments, or reserve bytes from the memory manager and keep their memory usage within that boundary. If unspecified, it will be derived to make up the configured fraction of the Total Flink Memory.
-
taskmanager.memory.segment-size
-
"32kb"
-
Size of memory buffers used by the network stack and the memory manager.
-
-
-
taskmanager.memory.shuffle.fraction
+
taskmanager.memory.network.fraction
0.1
-
Fraction of Total Flink Memory to be used as Shuffle Memory. Shuffle Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Shuffle Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Shuffle Memory can be explicitly specified by setting the min/max size to the same value.
+
Float
+
Fraction of Total Flink Memory to be used as Network Memory. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max size to the same value.
-
taskmanager.memory.shuffle.max
-
"1g"
-
Max Shuffle Memory size for TaskExecutors. Shuffle Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Shuffle Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Shuffle Memory can be explicitly specified by setting the min/max to the same value.
+
taskmanager.memory.network.max
+
1 gb
+
MemorySize
+
Max Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max to the same value.
-
taskmanager.memory.shuffle.min
-
"64m"
-
Min Shuffle Memory size for TaskExecutors. Shuffle Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Shuffle Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Shuffle Memory can be explicitly specified by setting the min/max to the same value.
+
taskmanager.memory.network.min
+
64 mb
+
MemorySize
+
Min Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max to the same value.
-
taskmanager.memory.task.heap.size
+
taskmanager.memory.process.size
(none)
-
Task Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for user code. If not specified, it will be derived as Total Flink Memory minus Framework Heap Memory, Task Off-Heap Memory, (On-Heap and Off-Heap) Managed Memory and Shuffle Memory.
+
MemorySize
+
Total Process Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. On containerized setups, this should be set to the container memory. See also 'taskmanager.memory.flink.size' for total Flink memory size configuration.
-
taskmanager.memory.task.off-heap.size
-
"0b"
-
Task Heap Memory size for TaskExecutors. This is the size of off heap memory (JVM direct memory or native memory) reserved for user code.
+
taskmanager.memory.segment-size
+
32 kb
+
MemorySize
+
Size of memory buffers used by the network stack and the memory manager.
-
taskmanager.memory.total-flink.size
+
taskmanager.memory.task.heap.size
(none)
-
Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Shuffle Memory.
+
MemorySize
+
Task Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for tasks. If not specified, it will be derived as Total Flink Memory minus Framework Heap Memory, Task Off-Heap Memory, Managed Memory and Network Memory.
-
taskmanager.memory.total-process.size
-
(none)
-
Total Process Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. On containerized setups, this should be set to the container memory.
+
taskmanager.memory.task.off-heap.size
+
0 bytes
+
MemorySize
+
Task Off-Heap Memory size for TaskExecutors. This is the size of off heap memory (JVM direct memory and native memory) reserved for tasks. The configured value will be fully counted when Flink calculates the JVM max direct memory size parameter.
Time window in milliseconds which defines the number of application attempt failures when restarting the AM. Failures which fall outside of this window are not being considered. Set this value to -1 in order to count globally. See here for more information.
yarn.application-attempts
(none)
+
String
Number of ApplicationMaster restarts. Note that that the entire Flink cluster will restart and the YARN Client will loose the connection. Also, the JobManager address will change and you’ll need to set the JM host:port manually. It is recommended to leave this option at 1.
yarn.application-master.port
"0"
+
String
With this configuration option, users can specify a port, a range of ports or a list of ports for the Application Master (and JobManager) RPC port. By default we recommend using the default value (0) to let the operating system choose an appropriate port. In particular when multiple AMs are running on the same physical host, fixed port assignments prevent the AM from starting. For example when running Flink on YARN on an environment with a restrictive firewall, this option allows specifying a range of allowed ports.
+
+
yarn.application.id
+
(none)
+
String
+
The YARN application id of the running yarn cluster. This is the YARN cluster where the pipeline is going to be executed.
+
+
+
yarn.application.name
+
(none)
+
String
+
A custom name for your YARN application.
+
+
+
yarn.application.node-label
+
(none)
+
String
+
Specify YARN node label for the YARN application.
+
yarn.application.priority
-1
+
Integer
A non-negative integer indicating the priority for submitting a Flink YARN application. It will only take effect if YARN priority scheduling setting is enabled. Larger integer corresponds with higher priority. If priority is negative or set to '-1'(default), Flink will unset yarn priority setting and use cluster default priority. Please refer to YARN's official documentation for specific settings required to enable priority scheduling for the targeted YARN version.
-
yarn.appmaster.rpc.address
+
yarn.application.queue
(none)
-
The hostname or address where the application master RPC system is listening.
+
String
+
The YARN queue on which to put the current pipeline.
-
yarn.appmaster.rpc.port
-
-1
-
The port where the application master RPC system is listening.
+
yarn.application.type
+
(none)
+
String
+
A custom type for your YARN application..
yarn.appmaster.vcores
1
+
Integer
The number of virtual cores (vcores) used by YARN application master.
yarn.containers.vcores
-1
+
Integer
The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise. In order for this parameter to be used your cluster must have CPU scheduling enabled. You can do this by setting the `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`.
+
+
yarn.file-replication
+
-1
+
Integer
+
Number of file replication of each local resource file. If it is not configured, Flink will use the default replication value in hadoop configuration.
+
+
+
yarn.flink-dist-jar
+
(none)
+
String
+
The location of the Flink dist jar.
+
yarn.heartbeat.container-request-interval
500
+
Integer
Time between heartbeats with the ResourceManager in milliseconds if Flink requests containers:
The lower this value is, the faster Flink will get notified about container allocations since requests and allocations are transmitted via heartbeats.
The lower this value is, the more excessive containers might get allocated which will eventually be released but put pressure on Yarn.
If you observe too many container allocations on the ResourceManager, then it is recommended to increase this value. See this link for more information.
yarn.heartbeat.interval
5
+
Integer
Time between heartbeats with the ResourceManager in seconds.
-
-
yarn.maximum-failed-containers
-
(none)
-
Maximum number of containers the system is going to reallocate in case of a failure.
-
yarn.per-job-cluster.include-user-jar
"ORDER"
-
Defines whether user-jars are included in the system class path for per-job-clusters as well as their positioning in the path. They can be positioned at the beginning ("FIRST"), at the end ("LAST"), or be positioned based on their name ("ORDER").
+
String
+
Defines whether user-jars are included in the system class path for per-job-clusters as well as their positioning in the path. They can be positioned at the beginning ("FIRST"), at the end ("LAST"), or be positioned based on their name ("ORDER"). "DISABLED" means the user-jars are excluded from the system class path.
yarn.properties-file.location
(none)
+
String
When a Flink job is submitted to YARN, the JobManager’s host and the number of available processing slots is written into a properties file, so that the Flink client is able to pick those details up. This configuration parameter allows changing the default location of that file (for example for environments sharing a Flink installation between users).
+
+
yarn.ship-directories
+
(none)
+
List<String>
+
A semicolon-separated list of directories to be shipped to the YARN cluster.
+
yarn.tags
(none)
+
String
A comma-separated list of tags to apply to the Flink YARN application.
+
+{{ content }}
diff --git a/docs/_layouts/plain.html b/docs/_layouts/plain.html
index 3f28fdf737782f57606cc325010cac6d0d80ebe4..e639e55a24b426afb5d7a65526e316806975d285 100644
--- a/docs/_layouts/plain.html
+++ b/docs/_layouts/plain.html
@@ -41,7 +41,7 @@ under the License.
{%- for p in active_pages %}
- {% capture title %}{% if p.nav-title %}{{ p.nav-title }}{% else %}{{ p.title }}{% endif %}{% endcapture %}
+ {% capture title %}{% if p.nav-title %}{{ p.nav-title }}{% else %}{{ p.title }}{% endif %}{% endcapture -%}
{%- if forloop.last == true %}
{{ title }}
{%- elsif p.nav-show_overview %}
diff --git a/docs/_layouts/redirect.html b/docs/_layouts/redirect.html
index 82bc2e2ea19b8e031d0f2db32100cdfa67f7eb1d..700e8aa5286cd3be9db8290c54f6eee436ced4d1 100644
--- a/docs/_layouts/redirect.html
+++ b/docs/_layouts/redirect.html
@@ -26,7 +26,7 @@ under the License.
Page '{{ page.title }}' Has Moved
- The {{ page.title }} has been moved. Redirecting to {{ site.baseurl }}{{ page.redirect }} in 1 second.
+ The page {{ page.title }} has been moved. Redirecting to {{ site.baseurl }}{{ page.redirect }} in 1 second.
{% else if page.language == "zh" %}
diff --git a/docs/build_docs.sh b/docs/build_docs.sh
index 1ab46f13af024a63b98e32376ae2cde4889227a8..51a5179a36b8816a1d9c94bef1827b0ce6976558 100755
--- a/docs/build_docs.sh
+++ b/docs/build_docs.sh
@@ -53,6 +53,8 @@ JEKYLL_CMD="build"
JEKYLL_CONFIG=""
+DOC_LANGUAGES="en zh"
+
# if -p flag is provided, serve site on localhost
# -i is like -p, but incremental (only rebuilds the modified file)
# -e builds only english documentation
@@ -64,7 +66,7 @@ while getopts "piez" opt; do
;;
i)
[[ `${RUBY} -v` =~ 'ruby 1' ]] && echo "Error: building the docs with the incremental option requires at least ruby 2.0" && exit 1
- JEKYLL_CMD="liveserve --baseurl= --watch --incremental"
+ JEKYLL_CMD="serve --baseurl= --watch --incremental"
;;
e)
JEKYLL_CONFIG="--config _config.yml,_config_dev_en.yml"
@@ -72,8 +74,34 @@ while getopts "piez" opt; do
z)
JEKYLL_CONFIG="--config _config.yml,_config_dev_zh.yml"
;;
+ *) echo "usage: $0 [-e|-z] [-i|-p]" >&2
+ exit 1 ;;
esac
done
# use 'bundle exec' to insert the local Ruby dependencies
-bundle exec jekyll ${JEKYLL_CMD} ${JEKYLL_CONFIG} --source "${DOCS_SRC}" --destination "${DOCS_DST}"
+
+if [ "${JEKYLL_CMD}" = "build" ] && [ -z "${JEKYLL_CONFIG}" ]; then
+ # run parallel builds for all languages if not serving or creating a single language only
+
+ # run processes and store pids
+ echo "Spawning parallel builds for languages: ${DOC_LANGUAGES}..."
+ pids=""
+ for lang in ${DOC_LANGUAGES}; do
+ bundle exec jekyll ${JEKYLL_CMD} --config _config.yml,_config_dev_${lang}.yml --source "${DOCS_SRC}" --destination "${DOCS_DST}_${lang}" &
+ pid=$!
+ pids="${pids} ${pid}"
+ done
+
+ # wait for all pids (since jekyll returns 0 even in case of failures, we do not parse exit codes)
+ wait ${pids}
+ rm -rf "${DOCS_DST}"
+ mkdir -p "${DOCS_DST}"
+ for lang in ${DOC_LANGUAGES}; do
+ cp -aln "${DOCS_DST}_${lang}/." "${DOCS_DST}"
+ rm -rf "${DOCS_DST}_${lang}"
+ done
+ exit 0
+else
+ bundle exec jekyll ${JEKYLL_CMD} ${JEKYLL_CONFIG} --source "${DOCS_SRC}" --destination "${DOCS_DST}"
+fi
diff --git a/docs/concepts/flink-architecture.md b/docs/concepts/flink-architecture.md
new file mode 100644
index 0000000000000000000000000000000000000000..3bc14dd098d04fa62ee2056d2122ed3dc3fe3ecd
--- /dev/null
+++ b/docs/concepts/flink-architecture.md
@@ -0,0 +1,132 @@
+---
+title: Flink Architecture
+nav-id: flink-architecture
+nav-pos: 4
+nav-title: Flink Architecture
+nav-parent_id: concepts
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+## Flink Applications and Flink Sessions
+
+`TODO: expand this section`
+
+{% top %}
+
+## Anatomy of a Flink Cluster
+
+`TODO: expand this section, especially about components of the Flink Master and
+container environments`
+
+The Flink runtime consists of two types of processes:
+
+ - The *Flink Master* coordinates the distributed execution. It schedules
+ tasks, coordinates checkpoints, coordinates recovery on failures, etc.
+
+ There is always at least one *Flink Master*. A high-availability setup
+ might have multiple *Flink Masters*, one of which one is always the
+ *leader*, and the others are *standby*.
+
+ - The *TaskManagers* (also called *workers*) execute the *tasks* (or more
+ specifically, the subtasks) of a dataflow, and buffer and exchange the data
+ *streams*.
+
+ There must always be at least one TaskManager.
+
+The Flink Master and TaskManagers can be started in various ways: directly on
+the machines as a [standalone cluster]({{ site.baseurl }}{% link
+ops/deployment/cluster_setup.md %}), in containers, or managed by resource
+frameworks like [YARN]({{ site.baseurl }}{% link ops/deployment/yarn_setup.md
+%}) or [Mesos]({{ site.baseurl }}{% link ops/deployment/mesos.md %}).
+TaskManagers connect to Flink Masters, announcing themselves as available, and
+are assigned work.
+
+The *client* is not part of the runtime and program execution, but is used to
+prepare and send a dataflow to the Flink Master. After that, the client can
+disconnect, or stay connected to receive progress reports. The client runs
+either as part of the Java/Scala program that triggers the execution, or in the
+command line process `./bin/flink run ...`.
+
+
+
+{% top %}
+
+## Tasks and Operator Chains
+
+For distributed execution, Flink *chains* operator subtasks together into
+*tasks*. Each task is executed by one thread. Chaining operators together into
+tasks is a useful optimization: it reduces the overhead of thread-to-thread
+handover and buffering, and increases overall throughput while decreasing
+latency. The chaining behavior can be configured; see the [chaining docs]({{
+site.baseurl }}{% link dev/stream/operators/index.md
+%}#task-chaining-and-resource-groups) for details.
+
+The sample dataflow in the figure below is executed with five subtasks, and
+hence with five parallel threads.
+
+
+
+{% top %}
+
+## Task Slots and Resources
+
+Each worker (TaskManager) is a *JVM process*, and may execute one or more
+subtasks in separate threads. To control how many tasks a worker accepts, a
+worker has so called **task slots** (at least one).
+
+Each *task slot* represents a fixed subset of resources of the TaskManager. A
+TaskManager with three slots, for example, will dedicate 1/3 of its managed
+memory to each slot. Slotting the resources means that a subtask will not
+compete with subtasks from other jobs for managed memory, but instead has a
+certain amount of reserved managed memory. Note that no CPU isolation happens
+here; currently slots only separate the managed memory of tasks.
+
+By adjusting the number of task slots, users can define how subtasks are
+isolated from each other. Having one slot per TaskManager means each task
+group runs in a separate JVM (which can be started in a separate container, for
+example). Having multiple slots means more subtasks share the same JVM. Tasks
+in the same JVM share TCP connections (via multiplexing) and heartbeat
+messages. They may also share data sets and data structures, thus reducing the
+per-task overhead.
+
+
+
+By default, Flink allows subtasks to share slots even if they are subtasks of
+different tasks, so long as they are from the same job. The result is that one
+slot may hold an entire pipeline of the job. Allowing this *slot sharing* has
+two main benefits:
+
+ - A Flink cluster needs exactly as many task slots as the highest parallelism
+ used in the job. No need to calculate how many tasks (with varying
+ parallelism) a program contains in total.
+
+ - It is easier to get better resource utilization. Without slot sharing, the
+ non-intensive *source/map()* subtasks would block as many resources as the
+ resource intensive *window* subtasks. With slot sharing, increasing the
+ base parallelism in our example from two to six yields full utilization of
+ the slotted resources, while making sure that the heavy subtasks are fairly
+ distributed among the TaskManagers.
+
+
+
+{% top %}
diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md
index 22f2c99ef2ef177bdfb2e83874383a5a7a763be3..0645786f8ba0ce2aecbba39a158356ed95ba0797 100644
--- a/docs/concepts/glossary.md
+++ b/docs/concepts/glossary.md
@@ -1,6 +1,6 @@
---
title: Glossary
-nav-pos: 3
+nav-pos: 10
nav-title: Glossary
nav-parent_id: concepts
---
@@ -79,8 +79,8 @@ whole [Flink Master](#flink-master) was called JobManager.
#### Logical Graph
A logical graph is a directed graph describing the high-level logic of a stream processing program.
-The nodes are [Operators](#operator) and the edges indicate input/output-relationships or
-data streams or data sets.
+The nodes are [Operators](#operator) and the edges indicate input/output-relationships of the
+operators and correspond to data streams or data sets.
#### Managed State
@@ -161,6 +161,6 @@ subsequent Tasks.
A Transformation is applied on one or more data streams or data sets and results in one or more
output data streams or data sets. A transformation might change a data stream or data set on a
per-record basis, but might also only change its partitioning or perform an aggregation. While
-[Operators](#operator) and [Functions](#function)) are the "physical" parts of Flink's API,
-Transformations are only an API concept. Specifically, most - but not all - transformations are
+[Operators](#operator) and [Functions](#function) are the "physical" parts of Flink's API,
+Transformations are only an API concept. Specifically, most transformations are
implemented by certain [Operators](#operator).
diff --git a/docs/concepts/glossary.zh.md b/docs/concepts/glossary.zh.md
index 5c5bd1d4cbfe6a0ccc3db4850143ac490ddb24b1..8efd2f101a7ebd365bc35d8735770970b34fcaf1 100644
--- a/docs/concepts/glossary.zh.md
+++ b/docs/concepts/glossary.zh.md
@@ -1,6 +1,6 @@
---
title: 词汇表
-nav-pos: 3
+nav-pos: 10
nav-title: 词汇表
nav-parent_id: concepts
---
diff --git a/docs/concepts/index.md b/docs/concepts/index.md
index 0c59f0462c7961be4d371c4c52718380b56ccf07..b236a5f4c8f9ae016c1e4c16628bababf4fc01d6 100644
--- a/docs/concepts/index.md
+++ b/docs/concepts/index.md
@@ -4,9 +4,9 @@ nav-id: concepts
nav-pos: 2
nav-title: ' Concepts'
nav-parent_id: root
-layout: redirect
-redirect: /concepts/programming-model.html
+nav-show_overview: true
permalink: /concepts/index.html
+always-expand: true
---
+
+Flink offers different levels of abstraction for developing streaming/batch applications.
+
+
+
+ - The lowest level abstraction simply offers **stateful streaming**. It is
+ embedded into the [DataStream API]({{ site.baseurl}}{% link
+ dev/datastream_api.md %}) via the [Process Function]({{ site.baseurl }}{%
+ link dev/stream/operators/process_function.md %}). It allows users freely
+ process events from one or more streams, and use consistent fault tolerant
+ *state*. In addition, users can register event time and processing time
+ callbacks, allowing programs to realize sophisticated computations.
+
+ - In practice, most applications would not need the above described low level
+ abstraction, but would instead program against the **Core APIs** like the
+ [DataStream API]({{ site.baseurl }}{% link dev/datastream_api.md %})
+ (bounded/unbounded streams) and the [DataSet API]({{ site.baseurl }}{% link
+ dev/batch/index.md %}) (bounded data sets). These fluent APIs offer the
+ common building blocks for data processing, like various forms of
+ user-specified transformations, joins, aggregations, windows, state, etc.
+ Data types processed in these APIs are represented as classes in the
+ respective programming languages.
+
+ The low level *Process Function* integrates with the *DataStream API*,
+ making it possible to go the lower level abstraction for certain operations
+ only. The *DataSet API* offers additional primitives on bounded data sets,
+ like loops/iterations.
+
+ - The **Table API** is a declarative DSL centered around *tables*, which may
+ be dynamically changing tables (when representing streams). The [Table
+ API]({{ site.baseurl }}{% link dev/table/index.md %}) follows the
+ (extended) relational model: Tables have a schema attached (similar to
+ tables in relational databases) and the API offers comparable operations,
+ such as select, project, join, group-by, aggregate, etc. Table API
+ programs declaratively define *what logical operation should be done*
+ rather than specifying exactly *how the code for the operation looks*.
+ Though the Table API is extensible by various types of user-defined
+ functions, it is less expressive than the *Core APIs*, but more concise to
+ use (less code to write). In addition, Table API programs also go through
+ an optimizer that applies optimization rules before execution.
+
+ One can seamlessly convert between tables and *DataStream*/*DataSet*,
+ allowing programs to mix *Table API* and with the *DataStream* and
+ *DataSet* APIs.
+
+ - The highest level abstraction offered by Flink is **SQL**. This abstraction
+ is similar to the *Table API* both in semantics and expressiveness, but
+ represents programs as SQL query expressions. The [SQL]({{ site.baseurl
+ }}{% link dev/table/index.md %}#sql) abstraction closely interacts with the
+ Table API, and SQL queries can be executed over tables defined in the
+ *Table API*.
+
+This _concepts_ section explains the basic concepts behind the different APIs,
+that is the concepts behind Flink as a stateful and timely stream processing
+system.
diff --git a/docs/concepts/index.zh.md b/docs/concepts/index.zh.md
index e3dfcc04c3c19ec27708ded5335bdc1dfa443465..2498a434abb5b07de86f134f6dd736bf85118bab 100644
--- a/docs/concepts/index.zh.md
+++ b/docs/concepts/index.zh.md
@@ -4,9 +4,9 @@ nav-id: concepts
nav-pos: 2
nav-title: ' 概念'
nav-parent_id: root
-layout: redirect
-redirect: /concepts/programming-model.html
+nav-show_overview: true
permalink: /concepts/index.html
+always-expand: true
---
+
+Flink offers different levels of abstraction for developing streaming/batch applications.
+
+
+
+ - The lowest level abstraction simply offers **stateful streaming**. It is
+ embedded into the [DataStream API]({{ site.baseurl}}{% link
+ dev/datastream_api.md %}) via the [Process Function]({{ site.baseurl }}{%
+ link dev/stream/operators/process_function.md %}). It allows users freely
+ process events from one or more streams, and use consistent fault tolerant
+ *state*. In addition, users can register event time and processing time
+ callbacks, allowing programs to realize sophisticated computations.
+
+ - In practice, most applications would not need the above described low level
+ abstraction, but would instead program against the **Core APIs** like the
+ [DataStream API]({{ site.baseurl }}{% link dev/datastream_api.md %})
+ (bounded/unbounded streams) and the [DataSet API]({{ site.baseurl }}{% link
+ dev/batch/index.md %}) (bounded data sets). These fluent APIs offer the
+ common building blocks for data processing, like various forms of
+ user-specified transformations, joins, aggregations, windows, state, etc.
+ Data types processed in these APIs are represented as classes in the
+ respective programming languages.
+
+ The low level *Process Function* integrates with the *DataStream API*,
+ making it possible to go the lower level abstraction for certain operations
+ only. The *DataSet API* offers additional primitives on bounded data sets,
+ like loops/iterations.
+
+ - The **Table API** is a declarative DSL centered around *tables*, which may
+ be dynamically changing tables (when representing streams). The [Table
+ API]({{ site.baseurl }}{% link dev/table/index.md %}) follows the
+ (extended) relational model: Tables have a schema attached (similar to
+ tables in relational databases) and the API offers comparable operations,
+ such as select, project, join, group-by, aggregate, etc. Table API
+ programs declaratively define *what logical operation should be done*
+ rather than specifying exactly *how the code for the operation looks*.
+ Though the Table API is extensible by various types of user-defined
+ functions, it is less expressive than the *Core APIs*, but more concise to
+ use (less code to write). In addition, Table API programs also go through
+ an optimizer that applies optimization rules before execution.
+
+ One can seamlessly convert between tables and *DataStream*/*DataSet*,
+ allowing programs to mix *Table API* and with the *DataStream* and
+ *DataSet* APIs.
+
+ - The highest level abstraction offered by Flink is **SQL**. This abstraction
+ is similar to the *Table API* both in semantics and expressiveness, but
+ represents programs as SQL query expressions. The [SQL]({{ site.baseurl
+ }}{% link dev/table/index.md %}#sql) abstraction closely interacts with the
+ Table API, and SQL queries can be executed over tables defined in the
+ *Table API*.
+
+This _concepts_ section explains the basic concepts behind the different APIs,
+that is the concepts behind Flink as a stateful and timely stream processing
+system.
diff --git a/docs/concepts/programming-model.md b/docs/concepts/programming-model.md
deleted file mode 100644
index 0ddafaa624713e496b3a5af33f3009a00b80f9e2..0000000000000000000000000000000000000000
--- a/docs/concepts/programming-model.md
+++ /dev/null
@@ -1,217 +0,0 @@
----
-title: Dataflow Programming Model
-nav-id: programming-model
-nav-pos: 1
-nav-title: Programming Model
-nav-parent_id: concepts
----
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## Levels of Abstraction
-
-Flink offers different levels of abstraction to develop streaming/batch applications.
-
-
-
- - The lowest level abstraction simply offers **stateful streaming**. It is embedded into the [DataStream API](../dev/datastream_api.html)
- via the [Process Function](../dev/stream/operators/process_function.html). It allows users freely process events from one or more streams,
- and use consistent fault tolerant *state*. In addition, users can register event time and processing time callbacks,
- allowing programs to realize sophisticated computations.
-
- - In practice, most applications would not need the above described low level abstraction, but would instead program against the
- **Core APIs** like the [DataStream API](../dev/datastream_api.html) (bounded/unbounded streams) and the [DataSet API](../dev/batch/index.html)
- (bounded data sets). These fluent APIs offer the common building blocks for data processing, like various forms of user-specified
- transformations, joins, aggregations, windows, state, etc. Data types processed in these APIs are represented as classes
- in the respective programming languages.
-
- The low level *Process Function* integrates with the *DataStream API*, making it possible to go the lower level abstraction
- for certain operations only. The *DataSet API* offers additional primitives on bounded data sets, like loops/iterations.
-
- - The **Table API** is a declarative DSL centered around *tables*, which may be dynamically changing tables (when representing streams).
- The [Table API](../dev/table/index.html) follows the (extended) relational model: Tables have a schema attached (similar to tables in relational databases)
- and the API offers comparable operations, such as select, project, join, group-by, aggregate, etc.
- Table API programs declaratively define *what logical operation should be done* rather than specifying exactly
- *how the code for the operation looks*. Though the Table API is extensible by various types of user-defined
- functions, it is less expressive than the *Core APIs*, but more concise to use (less code to write).
- In addition, Table API programs also go through an optimizer that applies optimization rules before execution.
-
- One can seamlessly convert between tables and *DataStream*/*DataSet*, allowing programs to mix *Table API* and with the *DataStream*
- and *DataSet* APIs.
-
- - The highest level abstraction offered by Flink is **SQL**. This abstraction is similar to the *Table API* both in semantics and
- expressiveness, but represents programs as SQL query expressions.
- The [SQL](../dev/table/index.html#sql) abstraction closely interacts with the Table API, and SQL queries can be executed over tables defined in the *Table API*.
-
-
-## Programs and Dataflows
-
-The basic building blocks of Flink programs are **streams** and **transformations**. (Note that the
-DataSets used in Flink's DataSet API are also streams internally -- more about that
-later.) Conceptually a *stream* is a (potentially never-ending) flow of data records, and a *transformation* is an
-operation that takes one or more streams as input, and produces one or more output streams as a
-result.
-
-When executed, Flink programs are mapped to **streaming dataflows**, consisting of **streams** and transformation **operators**.
-Each dataflow starts with one or more **sources** and ends in one or more **sinks**. The dataflows resemble
-arbitrary **directed acyclic graphs** *(DAGs)*. Although special forms of cycles are permitted via
-*iteration* constructs, for the most part we will gloss over this for simplicity.
-
-
-
-Often there is a one-to-one correspondence between the transformations in the programs and the operators
-in the dataflow. Sometimes, however, one transformation may consist of multiple transformation operators.
-
-Sources and sinks are documented in the [streaming connectors](../dev/connectors/index.html) and [batch connectors](../dev/batch/connectors.html) docs.
-Transformations are documented in [DataStream operators]({{ site.baseurl }}/dev/stream/operators/index.html) and [DataSet transformations](../dev/batch/dataset_transformations.html).
-
-{% top %}
-
-## Parallel Dataflows
-
-Programs in Flink are inherently parallel and distributed. During execution, a *stream* has one or more **stream partitions**,
-and each *operator* has one or more **operator subtasks**. The operator subtasks are independent of one another, and execute in different threads
-and possibly on different machines or containers.
-
-The number of operator subtasks is the **parallelism** of that particular operator. The parallelism of a stream
-is always that of its producing operator. Different operators of the same program may have different
-levels of parallelism.
-
-
-
-Streams can transport data between two operators in a *one-to-one* (or *forwarding*) pattern, or in a *redistributing* pattern:
-
- - **One-to-one** streams (for example between the *Source* and the *map()* operators in the figure
- above) preserve the partitioning and ordering of the
- elements. That means that subtask[1] of the *map()* operator will see the same elements in the same order as they
- were produced by subtask[1] of the *Source* operator.
-
- - **Redistributing** streams (as between *map()* and *keyBy/window* above, as well as between
- *keyBy/window* and *Sink*) change the partitioning of streams. Each *operator subtask* sends
- data to different target subtasks, depending on the selected transformation. Examples are
- *keyBy()* (which re-partitions by hashing the key), *broadcast()*, or *rebalance()* (which
- re-partitions randomly). In a *redistributing* exchange the ordering among the elements is
- only preserved within each pair of sending and receiving subtasks (for example, subtask[1]
- of *map()* and subtask[2] of *keyBy/window*). So in this example, the ordering within each key
- is preserved, but the parallelism does introduce non-determinism regarding the order in
- which the aggregated results for different keys arrive at the sink.
-
-Details about configuring and controlling parallelism can be found in the docs on [parallel execution](../dev/parallel.html).
-
-{% top %}
-
-## Windows
-
-Aggregating events (e.g., counts, sums) works differently on streams than in batch processing.
-For example, it is impossible to count all elements in a stream,
-because streams are in general infinite (unbounded). Instead, aggregates on streams (counts, sums, etc),
-are scoped by **windows**, such as *"count over the last 5 minutes"*, or *"sum of the last 100 elements"*.
-
-Windows can be *time driven* (example: every 30 seconds) or *data driven* (example: every 100 elements).
-One typically distinguishes different types of windows, such as *tumbling windows* (no overlap),
-*sliding windows* (with overlap), and *session windows* (punctuated by a gap of inactivity).
-
-
-
-More window examples can be found in this [blog post](https://flink.apache.org/news/2015/12/04/Introducing-windows.html).
-More details are in the [window docs](../dev/stream/operators/windows.html).
-
-{% top %}
-
-## Time
-
-When referring to time in a streaming program (for example to define windows), one can refer to different notions
-of time:
-
- - **Event Time** is the time when an event was created. It is usually described by a timestamp in the events,
- for example attached by the producing sensor, or the producing service. Flink accesses event timestamps
- via [timestamp assigners]({{ site.baseurl }}/dev/event_timestamps_watermarks.html).
-
- - **Ingestion time** is the time when an event enters the Flink dataflow at the source operator.
-
- - **Processing Time** is the local time at each operator that performs a time-based operation.
-
-
-
-More details on how to handle time are in the [event time docs]({{ site.baseurl }}/dev/event_time.html).
-
-{% top %}
-
-## Stateful Operations
-
-While many operations in a dataflow simply look at one individual *event at a time* (for example an event parser),
-some operations remember information across multiple events (for example window operators).
-These operations are called **stateful**.
-
-The state of stateful operations is maintained in what can be thought of as an embedded key/value store.
-The state is partitioned and distributed strictly together with the streams that are read by the
-stateful operators. Hence, access to the key/value state is only possible on *keyed streams*, after a *keyBy()* function,
-and is restricted to the values associated with the current event's key. Aligning the keys of streams and state
-makes sure that all state updates are local operations, guaranteeing consistency without transaction overhead.
-This alignment also allows Flink to redistribute the state and adjust the stream partitioning transparently.
-
-
-
-For more information, see the documentation on [state](../dev/stream/state/index.html).
-
-{% top %}
-
-## Checkpoints for Fault Tolerance
-
-Flink implements fault tolerance using a combination of **stream replay** and **checkpointing**. A
-checkpoint is related to a specific point in each of the input streams along with the corresponding state for each
-of the operators. A streaming dataflow can be resumed from a checkpoint while maintaining consistency *(exactly-once
-processing semantics)* by restoring the state of the operators and replaying the events from the
-point of the checkpoint.
-
-The checkpoint interval is a means of trading off the overhead of fault tolerance during execution with the recovery time (the number
-of events that need to be replayed).
-
-The description of the [fault tolerance internals]({{ site.baseurl }}/internals/stream_checkpointing.html) provides
-more information about how Flink manages checkpoints and related topics.
-Details about enabling and configuring checkpointing are in the [checkpointing API docs](../dev/stream/state/checkpointing.html).
-
-
-{% top %}
-
-## Batch on Streaming
-
-Flink executes [batch programs](../dev/batch/index.html) as a special case of streaming programs, where the streams are bounded (finite number of elements).
-A *DataSet* is treated internally as a stream of data. The concepts above thus apply to batch programs in the
-same way as well as they apply to streaming programs, with minor exceptions:
-
- - [Fault tolerance for batch programs](../dev/batch/fault_tolerance.html) does not use checkpointing.
- Recovery happens by fully replaying the streams.
- That is possible, because inputs are bounded. This pushes the cost more towards the recovery,
- but makes the regular processing cheaper, because it avoids checkpoints.
-
- - Stateful operations in the DataSet API use simplified in-memory/out-of-core data structures, rather than
- key/value indexes.
-
- - The DataSet API introduces special synchronized (superstep-based) iterations, which are only possible on
- bounded streams. For details, check out the [iteration docs]({{ site.baseurl }}/dev/batch/iterations.html).
-
-{% top %}
-
-## Next Steps
-
-Continue with the basic concepts in Flink's [Distributed Runtime](runtime.html).
diff --git a/docs/concepts/programming-model.zh.md b/docs/concepts/programming-model.zh.md
deleted file mode 100644
index 99253da5e9bfacc929f127e5b1a6b764bc50f309..0000000000000000000000000000000000000000
--- a/docs/concepts/programming-model.zh.md
+++ /dev/null
@@ -1,217 +0,0 @@
----
-title: 数据流编程模型
-nav-id: programming-model
-nav-pos: 1
-nav-title: 编程模型
-nav-parent_id: concepts
----
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## Levels of Abstraction
-
-Flink offers different levels of abstraction to develop streaming/batch applications.
-
-
-
- - The lowest level abstraction simply offers **stateful streaming**. It is embedded into the [DataStream API](../dev/datastream_api.html)
- via the [Process Function](../dev/stream/operators/process_function.html). It allows users freely process events from one or more streams,
- and use consistent fault tolerant *state*. In addition, users can register event time and processing time callbacks,
- allowing programs to realize sophisticated computations.
-
- - In practice, most applications would not need the above described low level abstraction, but would instead program against the
- **Core APIs** like the [DataStream API](../dev/datastream_api.html) (bounded/unbounded streams) and the [DataSet API](../dev/batch/index.html)
- (bounded data sets). These fluent APIs offer the common building blocks for data processing, like various forms of user-specified
- transformations, joins, aggregations, windows, state, etc. Data types processed in these APIs are represented as classes
- in the respective programming languages.
-
- The low level *Process Function* integrates with the *DataStream API*, making it possible to go the lower level abstraction
- for certain operations only. The *DataSet API* offers additional primitives on bounded data sets, like loops/iterations.
-
- - The **Table API** is a declarative DSL centered around *tables*, which may be dynamically changing tables (when representing streams).
- The [Table API](../dev/table/index.html) follows the (extended) relational model: Tables have a schema attached (similar to tables in relational databases)
- and the API offers comparable operations, such as select, project, join, group-by, aggregate, etc.
- Table API programs declaratively define *what logical operation should be done* rather than specifying exactly
- *how the code for the operation looks*. Though the Table API is extensible by various types of user-defined
- functions, it is less expressive than the *Core APIs*, but more concise to use (less code to write).
- In addition, Table API programs also go through an optimizer that applies optimization rules before execution.
-
- One can seamlessly convert between tables and *DataStream*/*DataSet*, allowing programs to mix *Table API* and with the *DataStream*
- and *DataSet* APIs.
-
- - The highest level abstraction offered by Flink is **SQL**. This abstraction is similar to the *Table API* both in semantics and
- expressiveness, but represents programs as SQL query expressions.
- The [SQL](../dev/table/index.html#sql) abstraction closely interacts with the Table API, and SQL queries can be executed over tables defined in the *Table API*.
-
-
-## Programs and Dataflows
-
-The basic building blocks of Flink programs are **streams** and **transformations**. (Note that the
-DataSets used in Flink's DataSet API are also streams internally -- more about that
-later.) Conceptually a *stream* is a (potentially never-ending) flow of data records, and a *transformation* is an
-operation that takes one or more streams as input, and produces one or more output streams as a
-result.
-
-When executed, Flink programs are mapped to **streaming dataflows**, consisting of **streams** and transformation **operators**.
-Each dataflow starts with one or more **sources** and ends in one or more **sinks**. The dataflows resemble
-arbitrary **directed acyclic graphs** *(DAGs)*. Although special forms of cycles are permitted via
-*iteration* constructs, for the most part we will gloss over this for simplicity.
-
-
-
-Often there is a one-to-one correspondence between the transformations in the programs and the operators
-in the dataflow. Sometimes, however, one transformation may consist of multiple transformation operators.
-
-Sources and sinks are documented in the [streaming connectors](../dev/connectors/index.html) and [batch connectors](../dev/batch/connectors.html) docs.
-Transformations are documented in [DataStream operators]({{ site.baseurl }}/dev/stream/operators/index.html) and [DataSet transformations](../dev/batch/dataset_transformations.html).
-
-{% top %}
-
-## Parallel Dataflows
-
-Programs in Flink are inherently parallel and distributed. During execution, a *stream* has one or more **stream partitions**,
-and each *operator* has one or more **operator subtasks**. The operator subtasks are independent of one another, and execute in different threads
-and possibly on different machines or containers.
-
-The number of operator subtasks is the **parallelism** of that particular operator. The parallelism of a stream
-is always that of its producing operator. Different operators of the same program may have different
-levels of parallelism.
-
-
-
-Streams can transport data between two operators in a *one-to-one* (or *forwarding*) pattern, or in a *redistributing* pattern:
-
- - **One-to-one** streams (for example between the *Source* and the *map()* operators in the figure
- above) preserve the partitioning and ordering of the
- elements. That means that subtask[1] of the *map()* operator will see the same elements in the same order as they
- were produced by subtask[1] of the *Source* operator.
-
- - **Redistributing** streams (as between *map()* and *keyBy/window* above, as well as between
- *keyBy/window* and *Sink*) change the partitioning of streams. Each *operator subtask* sends
- data to different target subtasks, depending on the selected transformation. Examples are
- *keyBy()* (which re-partitions by hashing the key), *broadcast()*, or *rebalance()* (which
- re-partitions randomly). In a *redistributing* exchange the ordering among the elements is
- only preserved within each pair of sending and receiving subtasks (for example, subtask[1]
- of *map()* and subtask[2] of *keyBy/window*). So in this example, the ordering within each key
- is preserved, but the parallelism does introduce non-determinism regarding the order in
- which the aggregated results for different keys arrive at the sink.
-
-Details about configuring and controlling parallelism can be found in the docs on [parallel execution](../dev/parallel.html).
-
-{% top %}
-
-## Windows
-
-Aggregating events (e.g., counts, sums) works differently on streams than in batch processing.
-For example, it is impossible to count all elements in a stream,
-because streams are in general infinite (unbounded). Instead, aggregates on streams (counts, sums, etc),
-are scoped by **windows**, such as *"count over the last 5 minutes"*, or *"sum of the last 100 elements"*.
-
-Windows can be *time driven* (example: every 30 seconds) or *data driven* (example: every 100 elements).
-One typically distinguishes different types of windows, such as *tumbling windows* (no overlap),
-*sliding windows* (with overlap), and *session windows* (punctuated by a gap of inactivity).
-
-
-
-More window examples can be found in this [blog post](https://flink.apache.org/news/2015/12/04/Introducing-windows.html).
-More details are in the [window docs](../dev/stream/operators/windows.html).
-
-{% top %}
-
-## Time
-
-When referring to time in a streaming program (for example to define windows), one can refer to different notions
-of time:
-
- - **Event Time** is the time when an event was created. It is usually described by a timestamp in the events,
- for example attached by the producing sensor, or the producing service. Flink accesses event timestamps
- via [timestamp assigners]({{ site.baseurl }}/dev/event_timestamps_watermarks.html).
-
- - **Ingestion time** is the time when an event enters the Flink dataflow at the source operator.
-
- - **Processing Time** is the local time at each operator that performs a time-based operation.
-
-
-
-More details on how to handle time are in the [event time docs]({{ site.baseurl }}/dev/event_time.html).
-
-{% top %}
-
-## Stateful Operations
-
-While many operations in a dataflow simply look at one individual *event at a time* (for example an event parser),
-some operations remember information across multiple events (for example window operators).
-These operations are called **stateful**.
-
-The state of stateful operations is maintained in what can be thought of as an embedded key/value store.
-The state is partitioned and distributed strictly together with the streams that are read by the
-stateful operators. Hence, access to the key/value state is only possible on *keyed streams*, after a *keyBy()* function,
-and is restricted to the values associated with the current event's key. Aligning the keys of streams and state
-makes sure that all state updates are local operations, guaranteeing consistency without transaction overhead.
-This alignment also allows Flink to redistribute the state and adjust the stream partitioning transparently.
-
-
-
-For more information, see the documentation on [state](../dev/stream/state/index.html).
-
-{% top %}
-
-## Checkpoints for Fault Tolerance
-
-Flink implements fault tolerance using a combination of **stream replay** and **checkpointing**. A
-checkpoint is related to a specific point in each of the input streams along with the corresponding state for each
-of the operators. A streaming dataflow can be resumed from a checkpoint while maintaining consistency *(exactly-once
-processing semantics)* by restoring the state of the operators and replaying the events from the
-point of the checkpoint.
-
-The checkpoint interval is a means of trading off the overhead of fault tolerance during execution with the recovery time (the number
-of events that need to be replayed).
-
-The description of the [fault tolerance internals]({{ site.baseurl }}/internals/stream_checkpointing.html) provides
-more information about how Flink manages checkpoints and related topics.
-Details about enabling and configuring checkpointing are in the [checkpointing API docs](../dev/stream/state/checkpointing.html).
-
-
-{% top %}
-
-## Batch on Streaming
-
-Flink executes [batch programs](../dev/batch/index.html) as a special case of streaming programs, where the streams are bounded (finite number of elements).
-A *DataSet* is treated internally as a stream of data. The concepts above thus apply to batch programs in the
-same way as well as they apply to streaming programs, with minor exceptions:
-
- - [Fault tolerance for batch programs](../dev/batch/fault_tolerance.html) does not use checkpointing.
- Recovery happens by fully replaying the streams.
- That is possible, because inputs are bounded. This pushes the cost more towards the recovery,
- but makes the regular processing cheaper, because it avoids checkpoints.
-
- - Stateful operations in the DataSet API use simplified in-memory/out-of-core data structures, rather than
- key/value indexes.
-
- - The DataSet API introduces special synchronized (superstep-based) iterations, which are only possible on
- bounded streams. For details, check out the [iteration docs]({{ site.baseurl }}/dev/batch/iterations.html).
-
-{% top %}
-
-## Next Steps
-
-Continue with the basic concepts in Flink's [Distributed Runtime](runtime.html).
diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md
deleted file mode 100644
index e6a2c51a3aec200fdc1a9cd6c3b83bf5eaaa9633..0000000000000000000000000000000000000000
--- a/docs/concepts/runtime.md
+++ /dev/null
@@ -1,127 +0,0 @@
----
-title: Distributed Runtime Environment
-nav-pos: 2
-nav-title: Distributed Runtime
-nav-parent_id: concepts
----
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## Tasks and Operator Chains
-
-For distributed execution, Flink *chains* operator subtasks together into *tasks*. Each task is executed by one thread.
-Chaining operators together into tasks is a useful optimization: it reduces the overhead of thread-to-thread
-handover and buffering, and increases overall throughput while decreasing latency.
-The chaining behavior can be configured; see the [chaining docs](../dev/stream/operators/#task-chaining-and-resource-groups) for details.
-
-The sample dataflow in the figure below is executed with five subtasks, and hence with five parallel threads.
-
-
-
-{% top %}
-
-## Job Managers, Task Managers, Clients
-
-The Flink runtime consists of two types of processes:
-
- - The **JobManagers** (also called *masters*) coordinate the distributed execution. They schedule tasks, coordinate
- checkpoints, coordinate recovery on failures, etc.
-
- There is always at least one Job Manager. A high-availability setup will have multiple JobManagers, one of
- which one is always the *leader*, and the others are *standby*.
-
- - The **TaskManagers** (also called *workers*) execute the *tasks* (or more specifically, the subtasks) of a dataflow,
- and buffer and exchange the data *streams*.
-
- There must always be at least one TaskManager.
-
-The JobManagers and TaskManagers can be started in various ways: directly on the machines as a [standalone cluster](../ops/deployment/cluster_setup.html), in
-containers, or managed by resource frameworks like [YARN](../ops/deployment/yarn_setup.html) or [Mesos](../ops/deployment/mesos.html).
-TaskManagers connect to JobManagers, announcing themselves as available, and are assigned work.
-
-The **client** is not part of the runtime and program execution, but is used to prepare and send a dataflow to the JobManager.
-After that, the client can disconnect, or stay connected to receive progress reports. The client runs either as part of the
-Java/Scala program that triggers the execution, or in the command line process `./bin/flink run ...`.
-
-
-
-{% top %}
-
-## Task Slots and Resources
-
-Each worker (TaskManager) is a *JVM process*, and may execute one or more subtasks in separate threads.
-To control how many tasks a worker accepts, a worker has so called **task slots** (at least one).
-
-Each *task slot* represents a fixed subset of resources of the TaskManager. A TaskManager with three slots, for example,
-will dedicate 1/3 of its managed memory to each slot. Slotting the resources means that a subtask will not
-compete with subtasks from other jobs for managed memory, but instead has a certain amount of reserved
-managed memory. Note that no CPU isolation happens here; currently slots only separate the managed memory of tasks.
-
-By adjusting the number of task slots, users can define how subtasks are isolated from each other.
-Having one slot per TaskManager means each task group runs in a separate JVM (which can be started in a
-separate container, for example). Having multiple slots
-means more subtasks share the same JVM. Tasks in the same JVM share TCP connections (via multiplexing) and
-heartbeat messages. They may also share data sets and data structures, thus reducing the per-task overhead.
-
-
-
-By default, Flink allows subtasks to share slots even if they are subtasks of different tasks, so long as
-they are from the same job. The result is that one slot may hold an entire pipeline of the
-job. Allowing this *slot sharing* has two main benefits:
-
- - A Flink cluster needs exactly as many task slots as the highest parallelism used in the job.
- No need to calculate how many tasks (with varying parallelism) a program contains in total.
-
- - It is easier to get better resource utilization. Without slot sharing, the non-intensive
- *source/map()* subtasks would block as many resources as the resource intensive *window* subtasks.
- With slot sharing, increasing the base parallelism in our example from two to six yields full utilization of the
- slotted resources, while making sure that the heavy subtasks are fairly distributed among the TaskManagers.
-
-
-
-The APIs also include a *[resource group](../dev/stream/operators/#task-chaining-and-resource-groups)* mechanism which can be used to prevent undesirable slot sharing.
-
-As a rule-of-thumb, a good default number of task slots would be the number of CPU cores.
-With hyper-threading, each slot then takes 2 or more hardware thread contexts.
-
-{% top %}
-
-## State Backends
-
-The exact data structures in which the key/values indexes are stored depends on the chosen [state backend](../ops/state/state_backends.html). One state backend
-stores data in an in-memory hash map, another state backend uses [RocksDB](http://rocksdb.org) as the key/value store.
-In addition to defining the data structure that holds the state, the state backends also implement the logic to
-take a point-in-time snapshot of the key/value state and store that snapshot as part of a checkpoint.
-
-
-
-{% top %}
-
-## Savepoints
-
-Programs written in the Data Stream API can resume execution from a **savepoint**. Savepoints allow both updating your programs and your Flink cluster without losing any state.
-
-[Savepoints](../ops/state/savepoints.html) are **manually triggered checkpoints**, which take a snapshot of the program and write it out to a state backend. They rely on the regular checkpointing mechanism for this. During execution programs are periodically snapshotted on the worker nodes and produce checkpoints. For recovery only the last completed checkpoint is needed and older checkpoints can be safely discarded as soon as a new one is completed.
-
-Savepoints are similar to these periodic checkpoints except that they are **triggered by the user** and **don't automatically expire** when newer checkpoints are completed. Savepoints can be created from the [command line](../ops/cli.html#savepoints) or when cancelling a job via the [REST API](../monitoring/rest_api.html#cancel-job-with-savepoint).
-
-{% top %}
diff --git a/docs/concepts/runtime.zh.md b/docs/concepts/runtime.zh.md
deleted file mode 100644
index 307c883730435b5dd8f092b1978590a50696269e..0000000000000000000000000000000000000000
--- a/docs/concepts/runtime.zh.md
+++ /dev/null
@@ -1,99 +0,0 @@
----
-title: 分布式运行时环境
-nav-pos: 2
-nav-title: 分布式运行时
-nav-parent_id: concepts
----
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## 任务和算子链
-
-分布式计算中，Flink 将算子（operator）的 subtask *链接（chain）*成 task。每个 task 由一个线程执行。把算子链接成 tasks 能够减少线程间切换和缓冲的开销，在降低延迟的同时提高了整体吞吐量。链接操作的配置详情可参考：[chaining docs](../dev/stream/operators/#task-chaining-and-resource-groups)
-
-下图的 dataflow 由五个 subtasks 执行，因此具有五个并行线程。
-
-
-
-{% top %}
-
-## Job Managers、Task Managers、客户端（Clients）
-
-Flink 运行时包含两类进程：
-
- - **JobManagers** （也称为 *masters*）协调分布式计算。它们负责调度任务、协调 checkpoints、协调故障恢复等。
-
- 每个 Job 至少会有一个 JobManager。高可用部署下会有多个 JobManagers，其中一个作为 *leader*，其余处于 *standby* 状态。
-
- - **TaskManagers**（也称为 *workers*）执行 dataflow 中的 *tasks*（准确来说是 subtasks ），并且缓存和交换数据 *streams*。
-
- 每个 Job 至少会有一个 TaskManager。
-
-JobManagers 和 TaskManagers 有多种启动方式：直接在机器上启动（该集群称为 [standalone cluster](../ops/deployment/cluster_setup.html)），在容器或资源管理框架，如 [YARN](../ops/deployment/yarn_setup.html) 或 [Mesos](../ops/deployment/mesos.html)，中启动。TaskManagers 连接到 JobManagers，通知后者自己可用，然后开始接手被分配的工作。
-
-**客户端**虽然不是运行时（runtime）和作业执行时的一部分，但它是被用作准备和提交 dataflow 到 JobManager 的。提交完成之后，客户端可以断开连接，也可以保持连接来接收进度报告。客户端既可以作为触发执行的 Java / Scala 程序的一部分，也可以在命令行进程中运行`./bin/flink run ...`。
-
-
-
-{% top %}
-
-## Task Slots 和资源
-
-每个 worker（TaskManager）都是一个 *JVM 进程*，并且可以在不同的线程中执行一个或多个 subtasks。为了控制 worker 接收 task 的数量，worker 拥有所谓的 **task slots** （至少一个）。
-
-每个 *task slots* 代表 TaskManager 的一份固定资源子集。例如，具有三个 slots 的 TaskManager 会将其管理的内存资源分成三等份给每个 slot。 划分资源意味着 subtask 之间不会竞争资源，但是也意味着它们只拥有固定的资源。注意这里并没有 CPU 隔离，当前 slots 之间只是划分任务的内存资源。
-
-通过调整 slot 的数量，用户可以决定 subtasks 的隔离方式。每个 TaskManager 有一个 slot 意味着每组 task 在一个单独的 JVM 中运行（例如，在一个单独的容器中启动）。拥有多个 slots 意味着多个 subtasks 共享同一个 JVM。 Tasks 在同一个 JVM 中共享 TCP 连接（通过多路复用技术）和心跳信息（heartbeat messages）。它们还可能共享数据集和数据结构，从而降低每个 task 的开销。
-
-
-
-默认情况下，Flink 允许 subtasks 共享 slots，即使它们是不同 tasks 的 subtasks，只要它们来自同一个 job。因此，一个 slot 可能会负责这个 job 的整个管道（pipeline）。允许 *slot sharing* 有两个好处：
-
- - Flink 集群需要与 job 中使用的最高并行度一样多的 slots。这样不需要计算作业总共包含多少个 tasks（具有不同并行度）。
-
- - 更好的资源利用率。在没有 slot sharing 的情况下，简单的 subtasks（*source/map()*）将会占用和复杂的 subtasks （*window*）一样多的资源。通过 slot sharing，将示例中的并行度从 2 增加到 6 可以充分利用 slot 的资源，同时确保繁重的 subtask 在 TaskManagers 之间公平地获取资源。
-
-
-
-APIs 还包含了 *[resource group](../dev/stream/operators/#task-chaining-and-resource-groups)* 机制，它可以用来防止不必要的 slot sharing。
-
-根据经验，合理的 slots 数量应该和 CPU 核数相同。在使用超线程（hyper-threading）时，每个 slot 将会占用 2 个或更多的硬件线程上下文（hardware thread contexts）。
-
-{% top %}
-
-## State Backends
-
-key/values 索引存储的数据结构取决于 [state backend](../ops/state/state_backends.html) 的选择。一类 state backend 将数据存储在内存的哈希映射中，另一类 state backend 使用 [RocksDB](http://rocksdb.org) 作为键/值存储。除了定义保存状态（state）的数据结构之外， state backend 还实现了获取键/值状态的时间点快照的逻辑，并将该快照存储为 checkpoint 的一部分。
-
-
-
-{% top %}
-
-## Savepoints
-
-用 Data Stream API 编写的程序可以从 **savepoint** 继续执行。Savepoints 允许在不丢失任何状态的情况下升级程序和 Flink 集群。
-
-[Savepoints](../ops/state/savepoints.html) 是**手动触发的 checkpoints**，它依靠常规的 checkpoint 机制获取程序的快照并将其写入 state backend。在执行期间，程序会定期在 worker 节点上创建快照并生成 checkpoints。对于恢复，Flink 仅需要最后完成的 checkpoint，而一旦完成了新的 checkpoint，旧的就可以被丢弃。
-
-Savepoints 类似于这些定期的 checkpoints，除了它们是**由用户触发**并且在新的 checkpoint 完成时**不会自动过期**。你可以通过[命令行](../ops/cli.html#savepoints) 或在取消一个 job 时通过 [REST API](../monitoring/rest_api.html#cancel-job-with-savepoint) 来创建 Savepoints。
-
-{% top %}
diff --git a/docs/concepts/stateful-stream-processing.md b/docs/concepts/stateful-stream-processing.md
new file mode 100644
index 0000000000000000000000000000000000000000..b7a06ba921bda444574e97357e82e7675cc749ac
--- /dev/null
+++ b/docs/concepts/stateful-stream-processing.md
@@ -0,0 +1,340 @@
+---
+title: Stateful Stream Processing
+nav-id: stateful-stream-processing
+nav-pos: 2
+nav-title: Stateful Stream Processing
+nav-parent_id: concepts
+---
+
+
+While many operations in a dataflow simply look at one individual *event at a
+time* (for example an event parser), some operations remember information
+across multiple events (for example window operators). These operations are
+called **stateful**.
+
+Some examples of stateful operations:
+
+ - When an application searches for certain event patterns, the state will
+ store the sequence of events encountered so far.
+ - When aggregating events per minute/hour/day, the state holds the pending
+ aggregates.
+ - When training a machine learning model over a stream of data points, the
+ state holds the current version of the model parameters.
+ - When historic data needs to be managed, the state allows efficient access
+ to events that occurred in the past.
+
+Flink needs to be aware of the state in order to make it fault tolerant using
+[checkpoints]({{ site.baseurl}}{% link dev/stream/state/checkpointing.md %})
+and [savepoints]({{ site.baseurl }}{%link ops/state/savepoints.md %}).
+
+Knowledge about the state also allows for rescaling Flink applications, meaning
+that Flink takes care of redistributing state across parallel instances.
+
+[Queryable state]({{ site.baseurl }}{% link dev/stream/state/queryable_state.md
+%}) allows you to access state from outside of Flink during runtime.
+
+When working with state, it might also be useful to read about [Flink's state
+backends]({{ site.baseurl }}{% link ops/state/state_backends.md %}). Flink
+provides different state backends that specify how and where state is stored.
+
+* This will be replaced by the TOC
+{:toc}
+
+## What is State?
+
+`TODO: expand this section`
+
+{% top %}
+
+## State in Stream & Batch Processing
+
+`TODO: What is this section about? Do we even need it?`
+
+{% top %}
+
+## Keyed State
+
+Keyed state is maintained in what can be thought of as an embedded key/value
+store. The state is partitioned and distributed strictly together with the
+streams that are read by the stateful operators. Hence, access to the key/value
+state is only possible on *keyed streams*, i.e. after a keyed/partitioned data
+exchange, and is restricted to the values associated with the current event's
+key. Aligning the keys of streams and state makes sure that all state updates
+are local operations, guaranteeing consistency without transaction overhead.
+This alignment also allows Flink to redistribute the state and adjust the
+stream partitioning transparently.
+
+
+
+Keyed State is further organized into so-called *Key Groups*. Key Groups are
+the atomic unit by which Flink can redistribute Keyed State; there are exactly
+as many Key Groups as the defined maximum parallelism. During execution each
+parallel instance of a keyed operator works with the keys for one or more Key
+Groups.
+
+## State Persistence
+
+Flink implements fault tolerance using a combination of **stream replay** and
+**checkpointing**. A checkpoint marks a specific point in each of the
+input streams along with the corresponding state for each of the operators. A
+streaming dataflow can be resumed from a checkpoint while maintaining
+consistency *(exactly-once processing semantics)* by restoring the state of the
+operators and replaying the records from the point of the checkpoint.
+
+The checkpoint interval is a means of trading off the overhead of fault
+tolerance during execution with the recovery time (the number of records that
+need to be replayed).
+
+The fault tolerance mechanism continuously draws snapshots of the distributed
+streaming data flow. For streaming applications with small state, these
+snapshots are very light-weight and can be drawn frequently without much impact
+on performance. The state of the streaming applications is stored at a
+configurable place, usually in a distributed file system.
+
+In case of a program failure (due to machine-, network-, or software failure),
+Flink stops the distributed streaming dataflow. The system then restarts the
+operators and resets them to the latest successful checkpoint. The input
+streams are reset to the point of the state snapshot. Any records that are
+processed as part of the restarted parallel dataflow are guaranteed to not have
+affected the previously checkpointed state.
+
+{% info Note %} By default, checkpointing is disabled. See [Checkpointing]({{
+site.baseurl }}{% link dev/stream/state/checkpointing.md %}) for details on how
+to enable and configure checkpointing.
+
+{% info Note %} For this mechanism to realize its full guarantees, the data
+stream source (such as message queue or broker) needs to be able to rewind the
+stream to a defined recent point. [Apache Kafka](http://kafka.apache.org) has
+this ability and Flink's connector to Kafka exploits this. See [Fault
+Tolerance Guarantees of Data Sources and Sinks]({{ site.baseurl }}{% link
+dev/connectors/guarantees.md %}) for more information about the guarantees
+provided by Flink's connectors.
+
+{% info Note %} Because Flink's checkpoints are realized through distributed
+snapshots, we use the words *snapshot* and *checkpoint* interchangeably. Often
+we also use the term *snapshot* to mean either *checkpoint* or *savepoint*.
+
+### Checkpointing
+
+The central part of Flink's fault tolerance mechanism is drawing consistent
+snapshots of the distributed data stream and operator state. These snapshots
+act as consistent checkpoints to which the system can fall back in case of a
+failure. Flink's mechanism for drawing these snapshots is described in
+"[Lightweight Asynchronous Snapshots for Distributed
+Dataflows](http://arxiv.org/abs/1506.08603)". It is inspired by the standard
+[Chandy-Lamport
+algorithm](http://research.microsoft.com/en-us/um/people/lamport/pubs/chandy.pdf)
+for distributed snapshots and is specifically tailored to Flink's execution
+model.
+
+Keep in mind that everything to do with checkpointing can be done
+asynchronously. The checkpoint barriers don't travel in lock step and
+operations can asynchronously snapshot their state.
+
+
+#### Barriers
+
+A core element in Flink's distributed snapshotting are the *stream barriers*.
+These barriers are injected into the data stream and flow with the records as
+part of the data stream. Barriers never overtake records, they flow strictly in
+line. A barrier separates the records in the data stream into the set of
+records that goes into the current snapshot, and the records that go into the
+next snapshot. Each barrier carries the ID of the snapshot whose records it
+pushed in front of it. Barriers do not interrupt the flow of the stream and are
+hence very lightweight. Multiple barriers from different snapshots can be in
+the stream at the same time, which means that various snapshots may happen
+concurrently.
+
+
+
+
+
+Stream barriers are injected into the parallel data flow at the stream sources.
+The point where the barriers for snapshot *n* are injected (let's call it
+Sn) is the position in the source stream up to which the
+snapshot covers the data. For example, in Apache Kafka, this position would be
+the last record's offset in the partition. This position Sn
+is reported to the *checkpoint coordinator* (Flink's JobManager).
+
+The barriers then flow downstream. When an intermediate operator has received a
+barrier for snapshot *n* from all of its input streams, it emits a barrier for
+snapshot *n* into all of its outgoing streams. Once a sink operator (the end of
+a streaming DAG) has received the barrier *n* from all of its input streams, it
+acknowledges that snapshot *n* to the checkpoint coordinator. After all sinks
+have acknowledged a snapshot, it is considered completed.
+
+Once snapshot *n* has been completed, the job will never again ask the source
+for records from before Sn, since at that point these records
+(and their descendant records) will have passed through the entire data flow
+topology.
+
+
+
+
+
+Operators that receive more than one input stream need to *align* the input
+streams on the snapshot barriers. The figure above illustrates this:
+
+ - As soon as the operator receives snapshot barrier *n* from an incoming
+ stream, it cannot process any further records from that stream until it has
+ received the barrier *n* from the other inputs as well. Otherwise, it would
+ mix records that belong to snapshot *n* and with records that belong to
+ snapshot *n+1*.
+ - Streams that report barrier *n* are temporarily set aside. Records that are
+ received from these streams are not processed, but put into an input
+ buffer.
+ - Once the last stream has received barrier *n*, the operator emits all
+ pending outgoing records, and then emits snapshot *n* barriers itself.
+ - After that, it resumes processing records from all input streams,
+ processing records from the input buffers before processing the records
+ from the streams.
+
+#### Snapshotting Operator State
+
+When operators contain any form of *state*, this state must be part of the
+snapshots as well.
+
+Operators snapshot their state at the point in time when they have received all
+snapshot barriers from their input streams, and before emitting the barriers to
+their output streams. At that point, all updates to the state from records
+before the barriers will have been made, and no updates that depend on records
+from after the barriers have been applied. Because the state of a snapshot may
+be large, it is stored in a configurable *[state backend]({{ site.baseurl }}{%
+link ops/state/state_backends.md %})*. By default, this is the JobManager's
+memory, but for production use a distributed reliable storage should be
+configured (such as HDFS). After the state has been stored, the operator
+acknowledges the checkpoint, emits the snapshot barrier into the output
+streams, and proceeds.
+
+The resulting snapshot now contains:
+
+ - For each parallel stream data source, the offset/position in the stream
+ when the snapshot was started
+ - For each operator, a pointer to the state that was stored as part of the
+ snapshot
+
+
+
+
+
+#### Recovery
+
+Recovery under this mechanism is straightforward: Upon a failure, Flink selects
+the latest completed checkpoint *k*. The system then re-deploys the entire
+distributed dataflow, and gives each operator the state that was snapshotted as
+part of checkpoint *k*. The sources are set to start reading the stream from
+position Sk. For example in Apache Kafka, that means telling
+the consumer to start fetching from offset Sk.
+
+If state was snapshotted incrementally, the operators start with the state of
+the latest full snapshot and then apply a series of incremental snapshot
+updates to that state.
+
+See [Restart Strategies]({{ site.baseurl }}{% link dev/task_failure_recovery.md
+%}#restart-strategies) for more information.
+
+### State Backends
+
+`TODO: expand this section`
+
+The exact data structures in which the key/values indexes are stored depends on
+the chosen [state backend]({{ site.baseurl }}{% link
+ops/state/state_backends.md %}). One state backend stores data in an in-memory
+hash map, another state backend uses [RocksDB](http://rocksdb.org) as the
+key/value store. In addition to defining the data structure that holds the
+state, the state backends also implement the logic to take a point-in-time
+snapshot of the key/value state and store that snapshot as part of a
+checkpoint. State backends can be configured without changing your application
+logic.
+
+
+
+{% top %}
+
+### Savepoints
+
+`TODO: expand this section`
+
+All programs that use checkpointing can resume execution from a **savepoint**.
+Savepoints allow both updating your programs and your Flink cluster without
+losing any state.
+
+[Savepoints]({{ site.baseurl }}{% link ops/state/savepoints.md %}) are
+**manually triggered checkpoints**, which take a snapshot of the program and
+write it out to a state backend. They rely on the regular checkpointing
+mechanism for this.
+
+Savepoints are similar to checkpoints except that they are
+**triggered by the user** and **don't automatically expire** when newer
+checkpoints are completed.
+
+{% top %}
+
+### Exactly Once vs. At Least Once
+
+The alignment step may add latency to the streaming program. Usually, this
+extra latency is on the order of a few milliseconds, but we have seen cases
+where the latency of some outliers increased noticeably. For applications that
+require consistently super low latencies (few milliseconds) for all records,
+Flink has a switch to skip the stream alignment during a checkpoint. Checkpoint
+snapshots are still drawn as soon as an operator has seen the checkpoint
+barrier from each input.
+
+When the alignment is skipped, an operator keeps processing all inputs, even
+after some checkpoint barriers for checkpoint *n* arrived. That way, the
+operator also processes elements that belong to checkpoint *n+1* before the
+state snapshot for checkpoint *n* was taken. On a restore, these records will
+occur as duplicates, because they are both included in the state snapshot of
+checkpoint *n*, and will be replayed as part of the data after checkpoint *n*.
+
+{% info Note %} Alignment happens only for operators with multiple predecessors
+(joins) as well as operators with multiple senders (after a stream
+repartitioning/shuffle). Because of that, dataflows with only embarrassingly
+parallel streaming operations (`map()`, `flatMap()`, `filter()`, ...) actually
+give *exactly once* guarantees even in *at least once* mode.
+
+{% top %}
+
+## End-to-end Exactly-Once Programs
+
+`TODO: add`
+
+## State and Fault Tolerance in Batch Programs
+
+Flink executes [batch programs](../dev/batch/index.html) as a special case of
+streaming programs, where the streams are bounded (finite number of elements).
+A *DataSet* is treated internally as a stream of data. The concepts above thus
+apply to batch programs in the same way as well as they apply to streaming
+programs, with minor exceptions:
+
+ - [Fault tolerance for batch programs](../dev/batch/fault_tolerance.html)
+ does not use checkpointing. Recovery happens by fully replaying the
+ streams. That is possible, because inputs are bounded. This pushes the
+ cost more towards the recovery, but makes the regular processing cheaper,
+ because it avoids checkpoints.
+
+ - Stateful operations in the DataSet API use simplified in-memory/out-of-core
+ data structures, rather than key/value indexes.
+
+ - The DataSet API introduces special synchronized (superstep-based)
+ iterations, which are only possible on bounded streams. For details, check
+ out the [iteration docs]({{ site.baseurl }}/dev/batch/iterations.html).
+
+{% top %}
diff --git a/docs/concepts/stream-processing.md b/docs/concepts/stream-processing.md
new file mode 100644
index 0000000000000000000000000000000000000000..aa094a7b2a8a4aabe9c1238cf5ba79d3477a89c7
--- /dev/null
+++ b/docs/concepts/stream-processing.md
@@ -0,0 +1,96 @@
+---
+title: Stream Processing
+nav-id: stream-processing
+nav-pos: 1
+nav-title: Stream Processing
+nav-parent_id: concepts
+---
+
+
+`TODO: Add introduction`
+* This will be replaced by the TOC
+{:toc}
+
+## A Unified System for Batch & Stream Processing
+
+`TODO`
+
+{% top %}
+
+## Programs and Dataflows
+
+The basic building blocks of Flink programs are **streams** and
+**transformations**. Conceptually a *stream* is a (potentially never-ending)
+flow of data records, and a *transformation* is an operation that takes one or
+more streams as input, and produces one or more output streams as a result.
+
+When executed, Flink programs are mapped to **streaming dataflows**, consisting
+of **streams** and transformation **operators**. Each dataflow starts with one
+or more **sources** and ends in one or more **sinks**. The dataflows resemble
+arbitrary **directed acyclic graphs** *(DAGs)*. Although special forms of
+cycles are permitted via *iteration* constructs, for the most part we will
+gloss over this for simplicity.
+
+
+
+Often there is a one-to-one correspondence between the transformations in the
+programs and the operators in the dataflow. Sometimes, however, one
+transformation may consist of multiple operators.
+
+{% top %}
+
+## Parallel Dataflows
+
+Programs in Flink are inherently parallel and distributed. During execution, a
+*stream* has one or more **stream partitions**, and each *operator* has one or
+more **operator subtasks**. The operator subtasks are independent of one
+another, and execute in different threads and possibly on different machines or
+containers.
+
+The number of operator subtasks is the **parallelism** of that particular
+operator. The parallelism of a stream is always that of its producing operator.
+Different operators of the same program may have different levels of
+parallelism.
+
+
+
+Streams can transport data between two operators in a *one-to-one* (or
+*forwarding*) pattern, or in a *redistributing* pattern:
+
+ - **One-to-one** streams (for example between the *Source* and the *map()*
+ operators in the figure above) preserve the partitioning and ordering of
+ the elements. That means that subtask[1] of the *map()* operator will see
+ the same elements in the same order as they were produced by subtask[1] of
+ the *Source* operator.
+
+ - **Redistributing** streams (as between *map()* and *keyBy/window* above, as
+ well as between *keyBy/window* and *Sink*) change the partitioning of
+ streams. Each *operator subtask* sends data to different target subtasks,
+ depending on the selected transformation. Examples are *keyBy()* (which
+ re-partitions by hashing the key), *broadcast()*, or *rebalance()* (which
+ re-partitions randomly). In a *redistributing* exchange the ordering among
+ the elements is only preserved within each pair of sending and receiving
+ subtasks (for example, subtask[1] of *map()* and subtask[2] of
+ *keyBy/window*). So in this example, the ordering within each key is
+ preserved, but the parallelism does introduce non-determinism regarding the
+ order in which the aggregated results for different keys arrive at the
+ sink.
+
+{% top %}
diff --git a/docs/concepts/timely-stream-processing.md b/docs/concepts/timely-stream-processing.md
new file mode 100644
index 0000000000000000000000000000000000000000..a67d6de0d9c5d02857a4875a0692bd5431cfa218
--- /dev/null
+++ b/docs/concepts/timely-stream-processing.md
@@ -0,0 +1,211 @@
+---
+title: Timely Stream Processing
+nav-id: timely-stream-processing
+nav-pos: 3
+nav-title: Timely Stream Processing
+nav-parent_id: concepts
+---
+
+
+`TODO: add introduction`
+
+* This will be replaced by the TOC
+{:toc}
+
+## Latency & Completeness
+
+`TODO: add these two sections`
+
+### Latency vs. Completeness in Batch & Stream Processing
+
+{% top %}
+
+## Notions of Time: Event Time and Processing Time
+
+When referring to time in a streaming program (for example to define windows),
+one can refer to different notions of *time*:
+
+- **Processing time:** Processing time refers to the system time of the machine
+ that is executing the respective operation.
+
+ When a streaming program runs on processing time, all time-based operations
+ (like time windows) will use the system clock of the machines that run the
+ respective operator. An hourly processing time window will include all
+ records that arrived at a specific operator between the times when the system
+ clock indicated the full hour. For example, if an application begins running
+ at 9:15am, the first hourly processing time window will include events
+ processed between 9:15am and 10:00am, the next window will include events
+ processed between 10:00am and 11:00am, and so on.
+
+ Processing time is the simplest notion of time and requires no coordination
+ between streams and machines. It provides the best performance and the
+ lowest latency. However, in distributed and asynchronous environments
+ processing time does not provide determinism, because it is susceptible to
+ the speed at which records arrive in the system (for example from the message
+ queue), to the speed at which the records flow between operators inside the
+ system, and to outages (scheduled, or otherwise).
+
+- **Event time:** Event time is the time that each individual event occurred on
+ its producing device. This time is typically embedded within the records
+ before they enter Flink, and that *event timestamp* can be extracted from
+ each record. In event time, the progress of time depends on the data, not on
+ any wall clocks. Event time programs must specify how to generate *Event Time
+ Watermarks*, which is the mechanism that signals progress in event time. This
+ watermarking mechanism is described in a later section,
+ [below](#event-time-and-watermarks).
+
+ In a perfect world, event time processing would yield completely consistent
+ and deterministic results, regardless of when events arrive, or their
+ ordering. However, unless the events are known to arrive in-order (by
+ timestamp), event time processing incurs some latency while waiting for
+ out-of-order events. As it is only possible to wait for a finite period of
+ time, this places a limit on how deterministic event time applications can
+ be.
+
+ Assuming all of the data has arrived, event time operations will behave as
+ expected, and produce correct and consistent results even when working with
+ out-of-order or late events, or when reprocessing historic data. For example,
+ an hourly event time window will contain all records that carry an event
+ timestamp that falls into that hour, regardless of the order in which they
+ arrive, or when they are processed. (See the section on [late
+ events](#late-elements) for more information.)
+
+ Note that sometimes when event time programs are processing live data in
+ real-time, they will use some *processing time* operations in order to
+ guarantee that they are progressing in a timely fashion.
+
+
+
+{% top %}
+
+## Event Time and Watermarks
+
+*Note: Flink implements many techniques from the Dataflow Model. For a good
+introduction to event time and watermarks, have a look at the articles below.*
+
+ - [Streaming
+ 101](https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101) by
+ Tyler Akidau
+ - The [Dataflow Model
+ paper](https://research.google.com/pubs/archive/43864.pdf)
+
+
+A stream processor that supports *event time* needs a way to measure the
+progress of event time. For example, a window operator that builds hourly
+windows needs to be notified when event time has passed beyond the end of an
+hour, so that the operator can close the window in progress.
+
+*Event time* can progress independently of *processing time* (measured by wall
+clocks). For example, in one program the current *event time* of an operator
+may trail slightly behind the *processing time* (accounting for a delay in
+receiving the events), while both proceed at the same speed. On the other
+hand, another streaming program might progress through weeks of event time with
+only a few seconds of processing, by fast-forwarding through some historic data
+already buffered in a Kafka topic (or another message queue).
+
+------
+
+The mechanism in Flink to measure progress in event time is **watermarks**.
+Watermarks flow as part of the data stream and carry a timestamp *t*. A
+*Watermark(t)* declares that event time has reached time *t* in that stream,
+meaning that there should be no more elements from the stream with a timestamp
+*t' <= t* (i.e. events with timestamps older or equal to the watermark).
+
+The figure below shows a stream of events with (logical) timestamps, and
+watermarks flowing inline. In this example the events are in order (with
+respect to their timestamps), meaning that the watermarks are simply periodic
+markers in the stream.
+
+
+
+Watermarks are crucial for *out-of-order* streams, as illustrated below, where
+the events are not ordered by their timestamps. In general a watermark is a
+declaration that by that point in the stream, all events up to a certain
+timestamp should have arrived. Once a watermark reaches an operator, the
+operator can advance its internal *event time clock* to the value of the
+watermark.
+
+
+
+Note that event time is inherited by a freshly created stream element (or
+elements) from either the event that produced them or from watermark that
+triggered creation of those elements.
+
+### Watermarks in Parallel Streams
+
+Watermarks are generated at, or directly after, source functions. Each parallel
+subtask of a source function usually generates its watermarks independently.
+These watermarks define the event time at that particular parallel source.
+
+As the watermarks flow through the streaming program, they advance the event
+time at the operators where they arrive. Whenever an operator advances its
+event time, it generates a new watermark downstream for its successor
+operators.
+
+Some operators consume multiple input streams; a union, for example, or
+operators following a *keyBy(...)* or *partition(...)* function. Such an
+operator's current event time is the minimum of its input streams' event times.
+As its input streams update their event times, so does the operator.
+
+The figure below shows an example of events and watermarks flowing through
+parallel streams, and operators tracking event time.
+
+
+
+## Lateness
+
+It is possible that certain elements will violate the watermark condition,
+meaning that even after the *Watermark(t)* has occurred, more elements with
+timestamp *t' <= t* will occur. In fact, in many real world setups, certain
+elements can be arbitrarily delayed, making it impossible to specify a time by
+which all elements of a certain event timestamp will have occurred.
+Furthermore, even if the lateness can be bounded, delaying the watermarks by
+too much is often not desirable, because it causes too much delay in the
+evaluation of event time windows.
+
+For this reason, streaming programs may explicitly expect some *late* elements.
+Late elements are elements that arrive after the system's event time clock (as
+signaled by the watermarks) has already passed the time of the late element's
+timestamp. See [Allowed Lateness]({{ site.baseurl }}{% link
+dev/stream/operators/windows.md %}#allowed-lateness) for more information on
+how to work with late elements in event time windows.
+
+## Windowing
+
+Aggregating events (e.g., counts, sums) works differently on streams than in
+batch processing. For example, it is impossible to count all elements in a
+stream, because streams are in general infinite (unbounded). Instead,
+aggregates on streams (counts, sums, etc), are scoped by **windows**, such as
+*"count over the last 5 minutes"*, or *"sum of the last 100 elements"*.
+
+Windows can be *time driven* (example: every 30 seconds) or *data driven*
+(example: every 100 elements). One typically distinguishes different types of
+windows, such as *tumbling windows* (no overlap), *sliding windows* (with
+overlap), and *session windows* (punctuated by a gap of inactivity).
+
+
+
+Please check out this [blog
+post](https://flink.apache.org/news/2015/12/04/Introducing-windows.html) for
+additional examples of windows or take a look a [window documentation]({{
+site.baseurl }}{% link dev/stream/operators/windows.md %}) of the DataStream
+API.
+
+{% top %}
diff --git a/docs/dev/api_concepts.md b/docs/dev/api_concepts.md
index 0da12faf23fcb93659f215cf7b532f1810746116..234b074562cabfed06778e727c8c044ca65dc107 100644
--- a/docs/dev/api_concepts.md
+++ b/docs/dev/api_concepts.md
@@ -216,9 +216,20 @@ Once you specified the complete program you need to **trigger the program execut
Depending on the type of the `ExecutionEnvironment` the execution will be triggered on your local
machine or submit your program for execution on a cluster.
-The `execute()` method is returning a `JobExecutionResult`, this contains execution
+The `execute()` method will wait for the job to finish and then return a `JobExecutionResult`, this contains execution
times and accumulator results.
+If you don't want to wait for the job to finish, you can trigger asynchronous job execution by calling
+`executeAysnc()` on the `StreamExecutionEnvironment`. It will return a `JobClient` with which you can
+communicate with the job you just submitted. For instance, here is how to implement the semantics
+of `execute()` by using `executeAsync()`.
+
+{% highlight java %}
+final JobClient jobClient = env.executeAsync();
+
+final JobExecutionResult jobExecutionResult = jobClient.getJobExecutionResult(userClassloader).get();
+{% endhighlight %}
+
Please see the [Streaming Guide]({{ site.baseurl }}/dev/datastream_api.html)
for information about streaming data sources and sink and for more in-depths information
about the supported transformations on DataStream.
diff --git a/docs/dev/api_concepts.zh.md b/docs/dev/api_concepts.zh.md
index 9bdbf5dc6b6997674145f95f80507267f48497f0..c71d635a871e4ab649389ac219166539c95ba0ed 100644
--- a/docs/dev/api_concepts.zh.md
+++ b/docs/dev/api_concepts.zh.md
@@ -160,6 +160,14 @@ print()
`execute()` 方法返回 `JobExecutionResult`，它包括执行耗时和一个累加器的结果。
+如果你不需要等待作业的结束，只是想要触发程序执行，你可以调用 `StreamExecutionEnvironment` 的 `executeAsync()` 方法。这个方法将返回一个 `JobClient` 对象，通过 `JobClient` 能够与程序对应的作业进行交互。作为例子，这里介绍通过 `executeAsync()` 实现与 `execute()` 相同行为的方法。
+
+{% highlight java %}
+final JobClient jobClient = env.executeAsync();
+
+final JobExecutionResult jobExecutionResult = jobClient.getJobExecutionResult(userClassloader).get();
+{% endhighlight %}
+
有关流数据的 source 和 sink 以及有关 DataStream 支持的转换操作的详细信息请参阅[流处理指南]({{ site.baseurl }}/zh/dev/datastream_api.html)。
有关批数据的 source 和 sink 以及有关 DataSet 支持的转换操作的详细信息请参阅[批处理指南]({{ site.baseurl }}/zh/dev/batch/index.html)。
diff --git a/docs/dev/batch/connectors.md b/docs/dev/batch/connectors.md
index 6221afe8ae2241859f5b02ba9d6a4ac26d5349c5..6a6fd79f04a3d82c6941cdf06acb64429f275c13 100644
--- a/docs/dev/batch/connectors.md
+++ b/docs/dev/batch/connectors.md
@@ -87,35 +87,35 @@ This example is using the `HadoopInputFormat` wrapper to use an existing Hadoop
1. Download and compile the `azure-tables-hadoop` project. The input format developed by the project is not yet available in Maven Central, therefore, we have to build the project ourselves.
Execute the following commands:
- {% highlight bash %}
- git clone https://github.com/mooso/azure-tables-hadoop.git
- cd azure-tables-hadoop
- mvn clean install
- {% endhighlight %}
+{% highlight bash %}
+git clone https://github.com/mooso/azure-tables-hadoop.git
+cd azure-tables-hadoop
+mvn clean install
+{% endhighlight %}
2. Setup a new Flink project using the quickstarts:
- {% highlight bash %}
- curl https://flink.apache.org/q/quickstart.sh | bash
- {% endhighlight %}
+{% highlight bash %}
+curl https://flink.apache.org/q/quickstart.sh | bash
+{% endhighlight %}
3. Add the following dependencies (in the `` section) to your `pom.xml` file:
- {% highlight xml %}
-
- org.apache.flink
- flink-hadoop-compatibility{{ site.scala_version_suffix }}
- {{site.version}}
-
-
- com.microsoft.hadoop
- microsoft-hadoop-azure
- 0.0.4
-
- {% endhighlight %}
-
- `flink-hadoop-compatibility` is a Flink package that provides the Hadoop input format wrappers.
- `microsoft-hadoop-azure` is adding the project we've build before to our project.
+{% highlight xml %}
+
+ org.apache.flink
+ flink-hadoop-compatibility{{ site.scala_version_suffix }}
+ {{site.version}}
+
+
+ com.microsoft.hadoop
+ microsoft-hadoop-azure
+ 0.0.4
+
+{% endhighlight %}
+
+`flink-hadoop-compatibility` is a Flink package that provides the Hadoop input format wrappers.
+`microsoft-hadoop-azure` is adding the project we've build before to our project.
The project is now prepared for starting to code. We recommend to import the project into an IDE, such as Eclipse or IntelliJ. (Import as a Maven project!).
Browse to the code of the `Job.java` file. Its an empty skeleton for a Flink job.
diff --git a/docs/dev/batch/connectors.zh.md b/docs/dev/batch/connectors.zh.md
index 41df05f9bd76536df972378d4efde30a10e45b4a..00b9344193e7f25d6e63d3c3cbac820beaba25fb 100644
--- a/docs/dev/batch/connectors.zh.md
+++ b/docs/dev/batch/connectors.zh.md
@@ -87,35 +87,35 @@ This example is using the `HadoopInputFormat` wrapper to use an existing Hadoop
1. Download and compile the `azure-tables-hadoop` project. The input format developed by the project is not yet available in Maven Central, therefore, we have to build the project ourselves.
Execute the following commands:
- {% highlight bash %}
- git clone https://github.com/mooso/azure-tables-hadoop.git
- cd azure-tables-hadoop
- mvn clean install
- {% endhighlight %}
+{% highlight bash %}
+git clone https://github.com/mooso/azure-tables-hadoop.git
+cd azure-tables-hadoop
+mvn clean install
+{% endhighlight %}
2. Setup a new Flink project using the quickstarts:
- {% highlight bash %}
- curl https://flink.apache.org/q/quickstart.sh | bash
- {% endhighlight %}
+{% highlight bash %}
+curl https://flink.apache.org/q/quickstart.sh | bash
+{% endhighlight %}
3. Add the following dependencies (in the `` section) to your `pom.xml` file:
- {% highlight xml %}
-
- org.apache.flink
- flink-hadoop-compatibility{{ site.scala_version_suffix }}
- {{site.version}}
-
-
- com.microsoft.hadoop
- microsoft-hadoop-azure
- 0.0.4
-
- {% endhighlight %}
-
- `flink-hadoop-compatibility` is a Flink package that provides the Hadoop input format wrappers.
- `microsoft-hadoop-azure` is adding the project we've build before to our project.
+{% highlight xml %}
+
+ org.apache.flink
+ flink-hadoop-compatibility{{ site.scala_version_suffix }}
+ {{site.version}}
+
+
+ com.microsoft.hadoop
+ microsoft-hadoop-azure
+ 0.0.4
+
+{% endhighlight %}
+
+`flink-hadoop-compatibility` is a Flink package that provides the Hadoop input format wrappers.
+`microsoft-hadoop-azure` is adding the project we've build before to our project.
The project is now prepared for starting to code. We recommend to import the project into an IDE, such as Eclipse or IntelliJ. (Import as a Maven project!).
Browse to the code of the `Job.java` file. Its an empty skeleton for a Flink job.
diff --git a/docs/dev/batch/examples.md b/docs/dev/batch/examples.md
index e04c44c233ba6b948877116f75278e2885e625f0..2f0f50fe8b646f902ac0b7ed75e49e36ce77641b 100644
--- a/docs/dev/batch/examples.md
+++ b/docs/dev/batch/examples.md
@@ -1,7 +1,7 @@
---
title: "Batch Examples"
nav-title: Batch Examples
-nav-parent_id: examples
+nav-parent_id: batch
nav-pos: 20
---
-
-This page contains a collection of best practices for Flink programmers on how to solve frequently encountered problems.
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## Parsing command line arguments and passing them around in your Flink application
-
-Almost all Flink applications, both batch and streaming, rely on external configuration parameters.
-They are used to specify input and output sources (like paths or addresses), system parameters (parallelism, runtime configuration), and application specific parameters (typically used within user functions).
-
-Flink provides a simple utility called `ParameterTool` to provide some basic tooling for solving these problems.
-Please note that you don't have to use the `ParameterTool` described here. Other frameworks such as [Commons CLI](https://commons.apache.org/proper/commons-cli/) and
-[argparse4j](http://argparse4j.sourceforge.net/) also work well with Flink.
-
-
-### Getting your configuration values into the `ParameterTool`
-
-The `ParameterTool` provides a set of predefined static methods for reading the configuration. The tool is internally expecting a `Map`, so it's very easy to integrate it with your own configuration style.
-
-
-#### From `.properties` files
-
-The following method will read a [Properties](https://docs.oracle.com/javase/tutorial/essential/environment/properties.html) file and provide the key/value pairs:
-{% highlight java %}
-String propertiesFilePath = "/home/sam/flink/myjob.properties";
-ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFilePath);
-
-File propertiesFile = new File(propertiesFilePath);
-ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFile);
-
-InputStream propertiesFileInputStream = new FileInputStream(file);
-ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFileInputStream);
-{% endhighlight %}
-
-
-#### From the command line arguments
-
-This allows getting arguments like `--input hdfs:///mydata --elements 42` from the command line.
-{% highlight java %}
-public static void main(String[] args) {
- ParameterTool parameter = ParameterTool.fromArgs(args);
- // .. regular code ..
-{% endhighlight %}
-
-
-#### From system properties
-
-When starting a JVM, you can pass system properties to it: `-Dinput=hdfs:///mydata`. You can also initialize the `ParameterTool` from these system properties:
-
-{% highlight java %}
-ParameterTool parameter = ParameterTool.fromSystemProperties();
-{% endhighlight %}
-
-
-### Using the parameters in your Flink program
-
-Now that we've got the parameters from somewhere (see above) we can use them in various ways.
-
-**Directly from the `ParameterTool`**
-
-The `ParameterTool` itself has methods for accessing the values.
-{% highlight java %}
-ParameterTool parameters = // ...
-parameter.getRequired("input");
-parameter.get("output", "myDefaultValue");
-parameter.getLong("expectedCount", -1L);
-parameter.getNumberOfParameters()
-// .. there are more methods available.
-{% endhighlight %}
-
-You can use the return values of these methods directly in the `main()` method of the client submitting the application.
-For example, you could set the parallelism of a operator like this:
-
-{% highlight java %}
-ParameterTool parameters = ParameterTool.fromArgs(args);
-int parallelism = parameters.get("mapParallelism", 2);
-DataSet> counts = text.flatMap(new Tokenizer()).setParallelism(parallelism);
-{% endhighlight %}
-
-Since the `ParameterTool` is serializable, you can pass it to the functions itself:
-
-{% highlight java %}
-ParameterTool parameters = ParameterTool.fromArgs(args);
-DataSet> counts = text.flatMap(new Tokenizer(parameters));
-{% endhighlight %}
-
-and then use it inside the function for getting values from the command line.
-
-#### Register the parameters globally
-
-Parameters registered as global job parameters in the `ExecutionConfig` can be accessed as configuration values from the JobManager web interface and in all functions defined by the user.
-
-Register the parameters globally:
-
-{% highlight java %}
-ParameterTool parameters = ParameterTool.fromArgs(args);
-
-// set up the execution environment
-final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-env.getConfig().setGlobalJobParameters(parameters);
-{% endhighlight %}
-
-Access them in any rich user function:
-
-{% highlight java %}
-public static final class Tokenizer extends RichFlatMapFunction> {
-
- @Override
- public void flatMap(String value, Collector> out) {
- ParameterTool parameters = (ParameterTool)
- getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
- parameters.getRequired("input");
- // .. do more ..
-{% endhighlight %}
-
-
-## Naming large TupleX types
-
-It is recommended to use POJOs (Plain old Java objects) instead of `TupleX` for data types with many fields.
-Also, POJOs can be used to give large `Tuple`-types a name.
-
-**Example**
-
-Instead of using:
-
-
-{% highlight java %}
-Tuple11 var = new ...;
-{% endhighlight %}
-
-
-It is much easier to create a custom type extending from the large Tuple type.
-
-{% highlight java %}
-CustomType var = new ...;
-
-public static class CustomType extends Tuple11 {
- // constructor matching super
-}
-{% endhighlight %}
-
-## Using Logback instead of Log4j
-
-**Note: This tutorial is applicable starting from Flink 0.10**
-
-Apache Flink is using [slf4j](http://www.slf4j.org/) as the logging abstraction in the code. Users are advised to use sfl4j as well in their user functions.
-
-Sfl4j is a compile-time logging interface that can use different logging implementations at runtime, such as [log4j](http://logging.apache.org/log4j/2.x/) or [Logback](http://logback.qos.ch/).
-
-Flink is depending on Log4j by default. This page describes how to use Flink with Logback. Users reported that they were also able to set up centralized logging with Graylog using this tutorial.
-
-To get a logger instance in the code, use the following code:
-
-
-{% highlight java %}
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MyClass implements MapFunction {
- private static final Logger LOG = LoggerFactory.getLogger(MyClass.class);
- // ...
-{% endhighlight %}
-
-
-### Use Logback when running Flink out of the IDE / from a Java application
-
-
-In all cases where classes are executed with a classpath created by a dependency manager such as Maven, Flink will pull log4j into the classpath.
-
-Therefore, you will need to exclude log4j from Flink's dependencies. The following description will assume a Maven project created from a [Flink quickstart](./projectsetup/java_api_quickstart.html).
-
-Change your projects `pom.xml` file like this:
-
-{% highlight xml %}
-
-
-
- ch.qos.logback
- logback-core
- 1.1.3
-
-
- ch.qos.logback
- logback-classic
- 1.1.3
-
-
-
-
- org.slf4j
- log4j-over-slf4j
- 1.7.7
-
-
-
- org.apache.flink
- flink-java
- {{ site.version }}
-
-
- log4j
- *
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
- org.apache.flink
- flink-streaming-java{{ site.scala_version_suffix }}
- {{ site.version }}
-
-
- log4j
- *
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
- org.apache.flink
- flink-clients{{ site.scala_version_suffix }}
- {{ site.version }}
-
-
- log4j
- *
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
-{% endhighlight %}
-
-The following changes were done in the `` section:
-
- * Exclude all `log4j` dependencies from all Flink dependencies: this causes Maven to ignore Flink's transitive dependencies to log4j.
- * Exclude the `slf4j-log4j12` artifact from Flink's dependencies: since we are going to use the slf4j to logback binding, we have to remove the slf4j to log4j binding.
- * Add the Logback dependencies: `logback-core` and `logback-classic`
- * Add dependencies for `log4j-over-slf4j`. `log4j-over-slf4j` is a tool which allows legacy applications which are directly using the Log4j APIs to use the Slf4j interface. Flink depends on Hadoop which is directly using Log4j for logging. Therefore, we need to redirect all logger calls from Log4j to Slf4j which is in turn logging to Logback.
-
-Please note that you need to manually add the exclusions to all new Flink dependencies you are adding to the pom file.
-
-You may also need to check if other (non-Flink) dependencies are pulling in log4j bindings. You can analyze the dependencies of your project with `mvn dependency:tree`.
-
-
-
-### Use Logback when running Flink on a cluster
-
-This tutorial is applicable when running Flink on YARN or as a standalone cluster.
-
-In order to use Logback instead of Log4j with Flink, you need to remove `log4j-1.2.xx.jar` and `sfl4j-log4j12-xxx.jar` from the `lib/` directory.
-
-Next, you need to put the following jar files into the `lib/` folder:
-
- * `logback-classic.jar`
- * `logback-core.jar`
- * `log4j-over-slf4j.jar`: This bridge needs to be present in the classpath for redirecting logging calls from Hadoop (which is using Log4j) to Slf4j.
-
-Note that you need to explicitly set the `lib/` directory when using a per-job YARN cluster.
-
-The command to submit Flink on YARN with a custom logger is: `./bin/flink run -yt $FLINK_HOME/lib <... remaining arguments ...>`
-
-{% top %}
diff --git a/docs/dev/best_practices.zh.md b/docs/dev/best_practices.zh.md
deleted file mode 100644
index b9d16860aa85c9ae4f14956be7960ac2c6df20ef..0000000000000000000000000000000000000000
--- a/docs/dev/best_practices.zh.md
+++ /dev/null
@@ -1,298 +0,0 @@
----
-title: "最佳实践"
-nav-parent_id: dev
-nav-pos: 90
----
-
-
-This page contains a collection of best practices for Flink programmers on how to solve frequently encountered problems.
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## Parsing command line arguments and passing them around in your Flink application
-
-Almost all Flink applications, both batch and streaming, rely on external configuration parameters.
-They are used to specify input and output sources (like paths or addresses), system parameters (parallelism, runtime configuration), and application specific parameters (typically used within user functions).
-
-Flink provides a simple utility called `ParameterTool` to provide some basic tooling for solving these problems.
-Please note that you don't have to use the `ParameterTool` described here. Other frameworks such as [Commons CLI](https://commons.apache.org/proper/commons-cli/) and
-[argparse4j](http://argparse4j.sourceforge.net/) also work well with Flink.
-
-
-### Getting your configuration values into the `ParameterTool`
-
-The `ParameterTool` provides a set of predefined static methods for reading the configuration. The tool is internally expecting a `Map`, so it's very easy to integrate it with your own configuration style.
-
-
-#### From `.properties` files
-
-The following method will read a [Properties](https://docs.oracle.com/javase/tutorial/essential/environment/properties.html) file and provide the key/value pairs:
-{% highlight java %}
-String propertiesFilePath = "/home/sam/flink/myjob.properties";
-ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFilePath);
-
-File propertiesFile = new File(propertiesFilePath);
-ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFile);
-
-InputStream propertiesFileInputStream = new FileInputStream(file);
-ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFileInputStream);
-{% endhighlight %}
-
-
-#### From the command line arguments
-
-This allows getting arguments like `--input hdfs:///mydata --elements 42` from the command line.
-{% highlight java %}
-public static void main(String[] args) {
- ParameterTool parameter = ParameterTool.fromArgs(args);
- // .. regular code ..
-{% endhighlight %}
-
-
-#### From system properties
-
-When starting a JVM, you can pass system properties to it: `-Dinput=hdfs:///mydata`. You can also initialize the `ParameterTool` from these system properties:
-
-{% highlight java %}
-ParameterTool parameter = ParameterTool.fromSystemProperties();
-{% endhighlight %}
-
-
-### Using the parameters in your Flink program
-
-Now that we've got the parameters from somewhere (see above) we can use them in various ways.
-
-**Directly from the `ParameterTool`**
-
-The `ParameterTool` itself has methods for accessing the values.
-{% highlight java %}
-ParameterTool parameters = // ...
-parameter.getRequired("input");
-parameter.get("output", "myDefaultValue");
-parameter.getLong("expectedCount", -1L);
-parameter.getNumberOfParameters()
-// .. there are more methods available.
-{% endhighlight %}
-
-You can use the return values of these methods directly in the `main()` method of the client submitting the application.
-For example, you could set the parallelism of a operator like this:
-
-{% highlight java %}
-ParameterTool parameters = ParameterTool.fromArgs(args);
-int parallelism = parameters.get("mapParallelism", 2);
-DataSet> counts = text.flatMap(new Tokenizer()).setParallelism(parallelism);
-{% endhighlight %}
-
-Since the `ParameterTool` is serializable, you can pass it to the functions itself:
-
-{% highlight java %}
-ParameterTool parameters = ParameterTool.fromArgs(args);
-DataSet> counts = text.flatMap(new Tokenizer(parameters));
-{% endhighlight %}
-
-and then use it inside the function for getting values from the command line.
-
-#### Register the parameters globally
-
-Parameters registered as global job parameters in the `ExecutionConfig` can be accessed as configuration values from the JobManager web interface and in all functions defined by the user.
-
-Register the parameters globally:
-
-{% highlight java %}
-ParameterTool parameters = ParameterTool.fromArgs(args);
-
-// set up the execution environment
-final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-env.getConfig().setGlobalJobParameters(parameters);
-{% endhighlight %}
-
-Access them in any rich user function:
-
-{% highlight java %}
-public static final class Tokenizer extends RichFlatMapFunction> {
-
- @Override
- public void flatMap(String value, Collector> out) {
- ParameterTool parameters = (ParameterTool)
- getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
- parameters.getRequired("input");
- // .. do more ..
-{% endhighlight %}
-
-
-## Naming large TupleX types
-
-It is recommended to use POJOs (Plain old Java objects) instead of `TupleX` for data types with many fields.
-Also, POJOs can be used to give large `Tuple`-types a name.
-
-**Example**
-
-Instead of using:
-
-
-{% highlight java %}
-Tuple11 var = new ...;
-{% endhighlight %}
-
-
-It is much easier to create a custom type extending from the large Tuple type.
-
-{% highlight java %}
-CustomType var = new ...;
-
-public static class CustomType extends Tuple11 {
- // constructor matching super
-}
-{% endhighlight %}
-
-## Using Logback instead of Log4j
-
-**Note: This tutorial is applicable starting from Flink 0.10**
-
-Apache Flink is using [slf4j](http://www.slf4j.org/) as the logging abstraction in the code. Users are advised to use sfl4j as well in their user functions.
-
-Sfl4j is a compile-time logging interface that can use different logging implementations at runtime, such as [log4j](http://logging.apache.org/log4j/2.x/) or [Logback](http://logback.qos.ch/).
-
-Flink is depending on Log4j by default. This page describes how to use Flink with Logback. Users reported that they were also able to set up centralized logging with Graylog using this tutorial.
-
-To get a logger instance in the code, use the following code:
-
-
-{% highlight java %}
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MyClass implements MapFunction {
- private static final Logger LOG = LoggerFactory.getLogger(MyClass.class);
- // ...
-{% endhighlight %}
-
-
-### Use Logback when running Flink out of the IDE / from a Java application
-
-
-In all cases where classes are executed with a classpath created by a dependency manager such as Maven, Flink will pull log4j into the classpath.
-
-Therefore, you will need to exclude log4j from Flink's dependencies. The following description will assume a Maven project created from a [Flink quickstart](./projectsetup/java_api_quickstart.html).
-
-Change your projects `pom.xml` file like this:
-
-{% highlight xml %}
-
-
-
- ch.qos.logback
- logback-core
- 1.1.3
-
-
- ch.qos.logback
- logback-classic
- 1.1.3
-
-
-
-
- org.slf4j
- log4j-over-slf4j
- 1.7.7
-
-
-
- org.apache.flink
- flink-java
- {{ site.version }}
-
-
- log4j
- *
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
- org.apache.flink
- flink-streaming-java{{ site.scala_version_suffix }}
- {{ site.version }}
-
-
- log4j
- *
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
- org.apache.flink
- flink-clients{{ site.scala_version_suffix }}
- {{ site.version }}
-
-
- log4j
- *
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
-{% endhighlight %}
-
-The following changes were done in the `` section:
-
- * Exclude all `log4j` dependencies from all Flink dependencies: this causes Maven to ignore Flink's transitive dependencies to log4j.
- * Exclude the `slf4j-log4j12` artifact from Flink's dependencies: since we are going to use the slf4j to logback binding, we have to remove the slf4j to log4j binding.
- * Add the Logback dependencies: `logback-core` and `logback-classic`
- * Add dependencies for `log4j-over-slf4j`. `log4j-over-slf4j` is a tool which allows legacy applications which are directly using the Log4j APIs to use the Slf4j interface. Flink depends on Hadoop which is directly using Log4j for logging. Therefore, we need to redirect all logger calls from Log4j to Slf4j which is in turn logging to Logback.
-
-Please note that you need to manually add the exclusions to all new Flink dependencies you are adding to the pom file.
-
-You may also need to check if other (non-Flink) dependencies are pulling in log4j bindings. You can analyze the dependencies of your project with `mvn dependency:tree`.
-
-
-
-### Use Logback when running Flink on a cluster
-
-This tutorial is applicable when running Flink on YARN or as a standalone cluster.
-
-In order to use Logback instead of Log4j with Flink, you need to remove `log4j-1.2.xx.jar` and `sfl4j-log4j12-xxx.jar` from the `lib/` directory.
-
-Next, you need to put the following jar files into the `lib/` folder:
-
- * `logback-classic.jar`
- * `logback-core.jar`
- * `log4j-over-slf4j.jar`: This bridge needs to be present in the classpath for redirecting logging calls from Hadoop (which is using Log4j) to Slf4j.
-
-Note that you need to explicitly set the `lib/` directory when using a per-job YARN cluster.
-
-The command to submit Flink on YARN with a custom logger is: `./bin/flink run -yt $FLINK_HOME/lib <... remaining arguments ...>`
-
-{% top %}
diff --git a/docs/dev/connectors/elasticsearch.md b/docs/dev/connectors/elasticsearch.md
index 23cdb8e26945aee34d21e9fea4586a1d86966cfd..7774d92a1d6c63c7e3f90371cced9edc26c6bb53 100644
--- a/docs/dev/connectors/elasticsearch.md
+++ b/docs/dev/connectors/elasticsearch.md
@@ -40,11 +40,6 @@ of the Elasticsearch installation:
-
{% highlight scala %}
import org.apache.flink.api.common.functions.RuntimeContext
import org.apache.flink.streaming.api.datastream.DataStream
@@ -255,15 +250,17 @@ httpHosts.add(new HttpHost("10.2.3.1", 9200, "http"))
val esSinkBuilder = new ElasticsearchSink.Builder[String](
httpHosts,
new ElasticsearchSinkFunction[String] {
- def createIndexRequest(element: String): IndexRequest = {
- val json = new java.util.HashMap[String, String]
- json.put("data", element)
-
- return Requests.indexRequest()
- .index("my-index")
- .type("my-type")
- .source(json)
- }
+ def process(element: String, ctx: RuntimeContext, indexer: RequestIndexer) {
+ val json = new java.util.HashMap[String, String]
+ json.put("data", element)
+
+ val rqst: IndexRequest = Requests.indexRequest
+ .index("my-index")
+ .`type`("my-type")
+ .source(json)
+
+ indexer.add(rqst)
+ }
}
)
diff --git a/docs/dev/connectors/guarantees.md b/docs/dev/connectors/guarantees.md
index cb5bffe745b31a938e869df7a3dc92086f8c6a6f..0749dbfe6fbd733746be6fa46d564476a29fea15 100644
--- a/docs/dev/connectors/guarantees.md
+++ b/docs/dev/connectors/guarantees.md
@@ -124,7 +124,7 @@ state updates) of Flink coupled with bundled sinks:
File sinks
-
at least once
+
exactly once
diff --git a/docs/dev/connectors/guarantees.zh.md b/docs/dev/connectors/guarantees.zh.md
index ef8a72bdceb4ae6e87368d3f788b473b695b3648..cd9c52d8f5ce00e46484ca034a1e73bf0a6cf611 100644
--- a/docs/dev/connectors/guarantees.zh.md
+++ b/docs/dev/connectors/guarantees.zh.md
@@ -124,7 +124,7 @@ state updates) of Flink coupled with bundled sinks:
File sinks
-
at least once
+
exactly once
diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md
index 3893dec34c1491cde418c4689e7f1778f9694117..d084808c01f8da7a7dc9619a1141aa8cb9f3ba24 100644
--- a/docs/dev/connectors/kafka.md
+++ b/docs/dev/connectors/kafka.md
@@ -34,7 +34,7 @@ exactly-once processing semantics. To achieve that, Flink does not purely rely o
offset tracking, but tracks and checkpoints these offsets internally as well.
Please pick a package (maven artifact id) and class name for your use-case and environment.
-For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is appropriate.
+For most users, the `FlinkKafkaConsumer010` (part of `flink-connector-kafka`) is appropriate.
@@ -48,22 +48,6 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is
-
@@ -121,7 +105,7 @@ Starting with Flink 1.7, there is a new universal Kafka connector that does not
Rather, it tracks the latest version of Kafka at the time of the Flink release.
If your Kafka broker version is 1.0.0 or newer, you should use this Kafka connector.
-If you use an older version of Kafka (0.11, 0.10, 0.9, or 0.8), you should use the connector corresponding to the broker version.
+If you use an older version of Kafka (0.11 or 0.10), you should use the connector corresponding to the broker version.
### Compatibility
@@ -156,7 +140,7 @@ except of dropping specific Kafka version from the module and class names.
## Kafka Consumer
-Flink's Kafka consumer is called `FlinkKafkaConsumer08` (or 09 for Kafka 0.9.0.x versions, etc.
+Flink's Kafka consumer is called `FlinkKafkaConsumer010` (or 011 for Kafka 0.11.0.x versions, etc.
or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). It provides access to one or more Kafka topics.
The constructor accepts the following arguments:
@@ -166,7 +150,6 @@ The constructor accepts the following arguments:
3. Properties for the Kafka consumer.
The following properties are required:
- "bootstrap.servers" (comma separated list of Kafka brokers)
- - "zookeeper.connect" (comma separated list of Zookeeper servers) (**only required for Kafka 0.8**)
- "group.id" the id of the consumer group
Example:
@@ -176,22 +159,18 @@ Example:
{% highlight java %}
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181");
properties.setProperty("group.id", "test");
DataStream stream = env
- .addSource(new FlinkKafkaConsumer08<>("topic", new SimpleStringSchema(), properties));
+ .addSource(new FlinkKafkaConsumer010<>("topic", new SimpleStringSchema(), properties));
{% endhighlight %}
{% highlight scala %}
val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181")
properties.setProperty("group.id", "test")
stream = env
- .addSource(new FlinkKafkaConsumer08[String]("topic", new SimpleStringSchema(), properties))
+ .addSource(new FlinkKafkaConsumer010[String]("topic", new SimpleStringSchema(), properties))
.print()
{% endhighlight %}
@@ -276,7 +255,7 @@ Example:
{% highlight java %}
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-FlinkKafkaConsumer08 myConsumer = new FlinkKafkaConsumer08<>(...);
+FlinkKafkaConsumer010 myConsumer = new FlinkKafkaConsumer010<>(...);
myConsumer.setStartFromEarliest(); // start from the earliest record possible
myConsumer.setStartFromLatest(); // start from the latest record
myConsumer.setStartFromTimestamp(...); // start from specified epoch timestamp (milliseconds)
@@ -290,7 +269,7 @@ DataStream stream = env.addSource(myConsumer);
{% highlight scala %}
val env = StreamExecutionEnvironment.getExecutionEnvironment()
-val myConsumer = new FlinkKafkaConsumer08[String](...)
+val myConsumer = new FlinkKafkaConsumer010[String](...)
myConsumer.setStartFromEarliest() // start from the earliest record possible
myConsumer.setStartFromLatest() // start from the latest record
myConsumer.setStartFromTimestamp(...) // start from specified epoch timestamp (milliseconds)
@@ -306,7 +285,7 @@ All versions of the Flink Kafka Consumer have the above explicit configuration m
* `setStartFromGroupOffsets` (default behaviour): Start reading partitions from
the consumer group's (`group.id` setting in the consumer properties) committed
- offsets in Kafka brokers (or Zookeeper for Kafka 0.8). If offsets could not be
+ offsets in Kafka brokers. If offsets could not be
found for a partition, the `auto.offset.reset` setting in the properties will be used.
* `setStartFromEarliest()` / `setStartFromLatest()`: Start from the earliest / latest
record. Under these modes, committed offsets in Kafka will be ignored and
@@ -437,7 +416,7 @@ val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
properties.setProperty("group.id", "test")
-val myConsumer = new FlinkKafkaConsumer08[String](
+val myConsumer = new FlinkKafkaConsumer010[String](
java.util.regex.Pattern.compile("test-topic-[0-9]"),
new SimpleStringSchema,
properties)
@@ -460,7 +439,7 @@ pattern.
### Kafka Consumers Offset Committing Behaviour Configuration
The Flink Kafka Consumer allows configuring the behaviour of how offsets
-are committed back to Kafka brokers (or Zookeeper in 0.8). Note that the
+are committed back to Kafka brokers. Note that the
Flink Kafka Consumer does not rely on the committed offsets for fault
tolerance guarantees. The committed offsets are only a means to expose
the consumer's progress for monitoring purposes.
@@ -471,8 +450,7 @@ whether or not checkpointing is enabled for the job.
- *Checkpointing disabled:* if checkpointing is disabled, the Flink Kafka
Consumer relies on the automatic periodic offset committing capability
of the internally used Kafka clients. Therefore, to disable or enable offset
- committing, simply set the `enable.auto.commit` (or `auto.commit.enable`
- for Kafka 0.8) / `auto.commit.interval.ms` keys to appropriate values
+ committing, simply set the `enable.auto.commit` / `auto.commit.interval.ms` keys to appropriate values
in the provided `Properties` configuration.
- *Checkpointing enabled:* if checkpointing is enabled, the Flink Kafka
@@ -502,12 +480,10 @@ can pass it to your consumer in the following way:
{% highlight java %}
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181");
properties.setProperty("group.id", "test");
-FlinkKafkaConsumer08 myConsumer =
- new FlinkKafkaConsumer08<>("topic", new SimpleStringSchema(), properties);
+FlinkKafkaConsumer010 myConsumer =
+ new FlinkKafkaConsumer010<>("topic", new SimpleStringSchema(), properties);
myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter());
DataStream stream = env
@@ -519,11 +495,9 @@ DataStream stream = env
{% highlight scala %}
val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181")
properties.setProperty("group.id", "test")
-val myConsumer = new FlinkKafkaConsumer08[String]("topic", new SimpleStringSchema(), properties)
+val myConsumer = new FlinkKafkaConsumer010[String]("topic", new SimpleStringSchema(), properties)
myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter())
stream = env
.addSource(myConsumer)
@@ -628,13 +602,9 @@ the `FlinkFixedPartitioner` is used instead.
### Kafka Producers and Fault Tolerance
-#### Kafka 0.8
-
-Before 0.9 Kafka did not provide any mechanisms to guarantee at-least-once or exactly-once semantics.
-
-#### Kafka 0.9 and 0.10
+#### Kafka 0.10
-With Flink's checkpointing enabled, the `FlinkKafkaProducer09` and `FlinkKafkaProducer010`
+With Flink's checkpointing enabled, the `FlinkKafkaProducer010`
can provide at-least-once delivery guarantees.
Besides enabling Flink's checkpointing, you should also configure the setter
@@ -652,7 +622,7 @@ methods `setLogFailuresOnly(boolean)` and `setFlushOnCheckpoint(boolean)` approp
been written to Kafka. This must be enabled for at-least-once.
In conclusion, the Kafka producer by default has at-least-once guarantees for versions
-0.9 and 0.10, with `setLogFailureOnly` set to `false` and `setFlushOnCheckpoint` set
+0.10, with `setLogFailureOnly` set to `false` and `setFlushOnCheckpoint` set
to `true`.
**Note**: By default, the number of retries is set to "0". This means that when `setLogFailuresOnly` is set to `false`,
@@ -764,14 +734,14 @@ config.setWriteTimestampToKafka(true);
Flink's Kafka connectors provide some metrics through Flink's [metrics system]({{ site.baseurl }}/monitoring/metrics.html) to analyze
the behavior of the connector.
The producers export Kafka's internal metrics through Flink's metric system for all supported versions. The consumers export
-all metrics starting from Kafka version 0.9. The Kafka documentation lists all exported metrics
+all metrics starting from Kafka version 0.10. The Kafka documentation lists all exported metrics
in its [documentation](http://kafka.apache.org/documentation/#selector_monitoring).
In addition to these metrics, all consumers expose the `current-offsets` and `committed-offsets` for each topic partition.
The `current-offsets` refers to the current offset in the partition. This refers to the offset of the last element that
we retrieved and emitted successfully. The `committed-offsets` is the last committed offset.
-The Kafka Consumers in Flink commit the offsets back to Zookeeper (Kafka 0.8) or the Kafka brokers (Kafka 0.9+). If checkpointing
+The Kafka Consumers in Flink commit the offsets back to the Kafka brokers (Kafka 0.10+). If checkpointing
is disabled, offsets are committed periodically.
With checkpointing, the commit happens once all operators in the streaming topology have confirmed that they've created a checkpoint of their state.
This provides users with at-least-once semantics for the offsets committed to Zookeeper or the broker. For offsets checkpointed to Flink, the system
@@ -782,7 +752,7 @@ the committed offset and the most recent offset in each partition is called the
the data slower from the topic than new data is added, the lag will increase and the consumer will fall behind.
For large production deployments we recommend monitoring that metric to avoid increasing latency.
-## Enabling Kerberos Authentication (for versions 0.9+ and above only)
+## Enabling Kerberos Authentication
Flink provides first-class support through the Kafka connector to authenticate to a Kafka installation
configured for Kerberos. Simply configure Flink in `flink-conf.yaml` to enable Kerberos authentication for Kafka like so:
diff --git a/docs/dev/connectors/kafka.zh.md b/docs/dev/connectors/kafka.zh.md
index 40c03f10c6511a6ee53991ec748c681ef829a401..b2cdc54ec56c0489d433d02bd65c5aea40798ff9 100644
--- a/docs/dev/connectors/kafka.zh.md
+++ b/docs/dev/connectors/kafka.zh.md
@@ -1,5 +1,5 @@
---
-title: "Apache Kafka Connector"
+title: "Apache Kafka 连接器"
nav-title: Kafka
nav-parent_id: connectors
nav-pos: 1
@@ -26,51 +26,30 @@ under the License.
* This will be replaced by the TOC
{:toc}
-This connector provides access to event streams served by [Apache Kafka](https://kafka.apache.org/).
+此连接器提供了访问 [Apache Kafka](https://kafka.apache.org/) 事件流的服务。
-Flink provides special Kafka Connectors for reading and writing data from/to Kafka topics.
-The Flink Kafka Consumer integrates with Flink's checkpointing mechanism to provide
-exactly-once processing semantics. To achieve that, Flink does not purely rely on Kafka's consumer group
-offset tracking, but tracks and checkpoints these offsets internally as well.
+Flink 提供了专门的 Kafka 连接器，向 Kafka topic 中读取或者写入数据。Flink Kafka Consumer 集成了 Flink 的 Checkpoint 机制，可提供 exactly-once 的处理语义。为此，Flink 并不完全依赖于跟踪 Kafka 消费组的偏移量，而是在内部跟踪和检查偏移量。
-Please pick a package (maven artifact id) and class name for your use-case and environment.
-For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is appropriate.
+根据你的用例和环境选择相应的包（maven artifact id）和类名。对于大多数用户来说，使用 `FlinkKafkaConsumer010`（ `flink-connector-kafka` 的一部分）是比较合适的。
@@ -78,7 +57,7 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is
FlinkKafkaConsumer011
FlinkKafkaProducer011
0.11.x
-
Since 0.11.x Kafka does not support scala 2.10. This connector supports Kafka transactional messaging to provide exactly once semantic for the producer.
@@ -87,17 +66,14 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is
FlinkKafkaProducer
>= 1.0.0
- This universal Kafka connector attempts to track the latest version of the Kafka client.
- The version of the client it uses may change between Flink releases. Starting with Flink 1.9 release, it uses the Kafka 2.2.0 client.
- Modern Kafka clients are backwards compatible with broker versions 0.10.0 or later.
- However for Kafka 0.11.x and 0.10.x versions, we recommend using dedicated
- flink-connector-kafka-0.11{{ site.scala_version_suffix }} and flink-connector-kafka-0.10{{ site.scala_version_suffix }} respectively.
+ 这个通用的 Kafka 连接器尽力与 Kafka client 的最新版本保持同步。该连接器使用的 Kafka client 版本可能会在 Flink 版本之间发生变化。从 Flink 1.9 版本开始，它使用 Kafka 2.2.0 client。当前 Kafka 客户端向后兼容 0.10.0 或更高版本的 Kafka broker。
+ 但是对于 Kafka 0.11.x 和 0.10.x 版本，我们建议你分别使用专用的 flink-connector-kafka-0.11{{ site.scala_version_suffix }} 和 flink-connector-kafka-0.10{{ site.scala_version_suffix }} 连接器。
-Then, import the connector in your maven project:
+接着，在你的 maven 项目中导入连接器：
{% highlight xml %}
@@ -107,40 +83,36 @@ Then, import the connector in your maven project:
{% endhighlight %}
-Note that the streaming connectors are currently not part of the binary distribution.
-See how to link with them for cluster execution [here]({{ site.baseurl}}/dev/projectsetup/dependencies.html).
-
-## Installing Apache Kafka
+请注意：目前流连接器还不是二进制分发的一部分。
+[在此处]({{ site.baseurl }}/zh/dev/projectsetup/dependencies.html)可以了解到如何链接它们以实现在集群中执行。
-* Follow the instructions from [Kafka's quickstart](https://kafka.apache.org/documentation.html#quickstart) to download the code and launch a server (launching a Zookeeper and a Kafka server is required every time before starting the application).
-* If the Kafka and Zookeeper servers are running on a remote machine, then the `advertised.host.name` setting in the `config/server.properties` file must be set to the machine's IP address.
+## 安装 Apache Kafka
-## Kafka 1.0.0+ Connector
+* 按照 [ Kafka 快速入门](https://kafka.apache.org/documentation.html#quickstart)的说明下载代码并启动 Kafka 服务器（每次启动应用程序之前都需要启动 Zookeeper 和 Kafka server）。
+* 如果 Kafka 和 Zookeeper 服务器运行在远端机器上，那么必须要将 `config/server.properties` 文件中的 `advertised.host.name`属性设置为远端设备的 IP 地址。
-Starting with Flink 1.7, there is a new universal Kafka connector that does not track a specific Kafka major version.
-Rather, it tracks the latest version of Kafka at the time of the Flink release.
+## Kafka 1.0.0+ 连接器
-If your Kafka broker version is 1.0.0 or newer, you should use this Kafka connector.
-If you use an older version of Kafka (0.11, 0.10, 0.9, or 0.8), you should use the connector corresponding to the broker version.
+从 Flink 1.7 开始，有一个新的通用 Kafka 连接器，它不跟踪特定的 Kafka 主版本。相反，它是在 Flink 发布时跟踪最新版本的 Kafka。
+如果你的 Kafka broker 版本是 1.0.0 或 更新的版本，你应该使用这个 Kafka 连接器。
+如果你使用的是 Kafka 的旧版本( 0.11 或 0.10 )，那么你应该使用与 Kafka broker 版本相对应的连接器。
-### Compatibility
+### 兼容性
-The universal Kafka connector is compatible with older and newer Kafka brokers through the compatibility guarantees of the Kafka client API and broker.
-It is compatible with broker versions 0.11.0 or newer, depending on the features used.
-For details on Kafka compatibility, please refer to the [Kafka documentation](https://kafka.apache.org/protocol.html#protocol_compatibility).
+通过 Kafka client API 和 broker 的兼容性保证，通用的 Kafka 连接器兼容较旧和较新的 Kafka broker。
+它兼容 Kafka broker 0.11.0 或者更高版本，具体兼容性取决于所使用的功能。有关 Kafka 兼容性的详细信息，请参考 [Kafka 文档](https://kafka.apache.org/protocol.html#protocol_compatibility)。
-### Migrating Kafka Connector from 0.11 to universal
+### 将 Kafka Connector 从 0.11 迁移到通用版本
-In order to perform the migration, see the [upgrading jobs and Flink versions guide]({{ site.baseurl }}/ops/upgrading.html)
-and:
-* Use Flink 1.9 or newer for the whole process.
-* Do not upgrade the Flink and operators at the same time.
-* Make sure that Kafka Consumer and/or Kafka Producer used in your job have assigned unique identifiers (`uid`):
-* Use stop with savepoint feature to take the savepoint (for example by using `stop --withSavepoint`)[CLI command]({{ site.baseurl }}/ops/cli.html).
+以便执行迁移，请参考 [升级 Jobs 和 Flink 版本指南]({{ site.baseurl }}/zh/ops/upgrading.html)：
+* 在全程中使用 Flink 1.9 或更新版本。
+* 不要同时升级 Flink 和 Operator。
+* 确保你的 Job 中所使用的 Kafka Consumer 和 Kafka Producer 分配了唯一的标识符（uid）。
+* 使用 stop with savepoint 的特性来执行 savepoint（例如，使用 `stop --withSavepoint`）[CLI 命令]({{ site.baseurl }}/zh/ops/cli.html)。
-### Usage
+### 用法
-To use the universal Kafka connector add a dependency to it:
+要使用通用的 Kafka 连接器，请为它添加依赖关系：
{% highlight xml %}
@@ -150,88 +122,69 @@ To use the universal Kafka connector add a dependency to it:
{% endhighlight %}
-Then instantiate the new source (`FlinkKafkaConsumer`) and sink (`FlinkKafkaProducer`).
-The API is backward compatible with the Kafka 0.11 connector,
-except of dropping specific Kafka version from the module and class names.
+然后，实例化 source（ `FlinkKafkaConsumer`）和 sink（ `FlinkKafkaProducer`）。除了从模块和类名中删除了特定的 Kafka 版本外，这个 API 向后兼容 Kafka 0.11 版本的 connector。
## Kafka Consumer
-Flink's Kafka consumer is called `FlinkKafkaConsumer08` (or 09 for Kafka 0.9.0.x versions, etc.
-or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). It provides access to one or more Kafka topics.
+Flink 的 Kafka consumer 称为 `FlinkKafkaConsumer010`（或适用于 Kafka 0.11.0.x 版本的 `FlinkKafkaConsumer011`，或适用于 Kafka >= 1.0.0 的版本的 `FlinkKafkaConsumer`）。它提供对一个或多个 Kafka topics 的访问。
-The constructor accepts the following arguments:
+构造函数接受以下参数：
-1. The topic name / list of topic names
-2. A DeserializationSchema / KafkaDeserializationSchema for deserializing the data from Kafka
-3. Properties for the Kafka consumer.
- The following properties are required:
- - "bootstrap.servers" (comma separated list of Kafka brokers)
- - "zookeeper.connect" (comma separated list of Zookeeper servers) (**only required for Kafka 0.8**)
- - "group.id" the id of the consumer group
+1. Topic 名称或者名称列表
+2. 用于反序列化 Kafka 数据的 DeserializationSchema 或者 KafkaDeserializationSchema
+3. Kafka 消费者的属性。需要以下属性：
+ - "bootstrap.servers"（以逗号分隔的 Kafka broker 列表）
+ - "group.id" 消费组 ID
-Example:
+示例：
{% highlight java %}
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181");
properties.setProperty("group.id", "test");
DataStream stream = env
- .addSource(new FlinkKafkaConsumer08<>("topic", new SimpleStringSchema(), properties));
+ .addSource(new FlinkKafkaConsumer010<>("topic", new SimpleStringSchema(), properties));
{% endhighlight %}
{% highlight scala %}
val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181")
properties.setProperty("group.id", "test")
stream = env
- .addSource(new FlinkKafkaConsumer08[String]("topic", new SimpleStringSchema(), properties))
+ .addSource(new FlinkKafkaConsumer010[String]("topic", new SimpleStringSchema(), properties))
.print()
{% endhighlight %}
-### The `DeserializationSchema`
+### `DeserializationSchema`
-The Flink Kafka Consumer needs to know how to turn the binary data in Kafka into Java/Scala objects. The
-`DeserializationSchema` allows users to specify such a schema. The `T deserialize(byte[] message)`
-method gets called for each Kafka message, passing the value from Kafka.
+Flink Kafka Consumer 需要知道如何将 Kafka 中的二进制数据转换为 Java 或者 Scala 对象。`DeserializationSchema` 允许用户指定这样的 schema，为每条 Kafka 消息调用 `T deserialize(byte[] message)` 方法，传递来自 Kafka 的值。
-It is usually helpful to start from the `AbstractDeserializationSchema`, which takes care of describing the
-produced Java/Scala type to Flink's type system. Users that implement a vanilla `DeserializationSchema` need
-to implement the `getProducedType(...)` method themselves.
+从 `AbstractDeserializationSchema` 开始通常很有帮助，它负责将生成的 Java 或 Scala 类型描述为 Flink 的类型系统。
+用户如果要自己去实现一个`DeserializationSchema`，需要自己去实现 `getProducedType(...)`方法。
-For accessing the key, value and metadata of the Kafka message, the `KafkaDeserializationSchema` has
-the following deserialize method `T deserialize(ConsumerRecord record)`.
+为了访问 Kafka 消息的 key、value 和元数据，`KafkaDeserializationSchema` 具有以下反序列化方法 `T deserialize(ConsumerRecord record)`。
-For convenience, Flink provides the following schemas:
+为了方便使用，Flink 提供了以下几种 schemas：
-1. `TypeInformationSerializationSchema` (and `TypeInformationKeyValueSerializationSchema`) which creates
- a schema based on a Flink's `TypeInformation`. This is useful if the data is both written and read by Flink.
- This schema is a performant Flink-specific alternative to other generic serialization approaches.
+1. `TypeInformationSerializationSchema`（和 `TypeInformationKeyValueSerializationSchema`) 基于 Flink 的 `TypeInformation` 创建 `schema`。
+ 如果该数据的读和写都发生在 Flink 中，那么这将是非常有用的。此 schema 是其他通用序列化方法的高性能 Flink 替代方案。
-2. `JsonDeserializationSchema` (and `JSONKeyValueDeserializationSchema`) which turns the serialized JSON
- into an ObjectNode object, from which fields can be accessed using `objectNode.get("field").as(Int/String/...)()`.
- The KeyValue objectNode contains a "key" and "value" field which contain all fields, as well as
- an optional "metadata" field that exposes the offset/partition/topic for this message.
+2. `JsonDeserializationSchema`（和 `JSONKeyValueDeserializationSchema`）将序列化的 JSON 转化为 ObjectNode 对象，可以使用 `objectNode.get("field").as(Int/String/...)()` 来访问某个字段。
+ KeyValue objectNode 包含一个含所有字段的 key 和 values 字段，以及一个可选的"metadata"字段，可以访问到消息的 offset、partition、topic 等信息。
-3. `AvroDeserializationSchema` which reads data serialized with Avro format using a statically provided schema. It can
- infer the schema from Avro generated classes (`AvroDeserializationSchema.forSpecific(...)`) or it can work with `GenericRecords`
- with a manually provided schema (with `AvroDeserializationSchema.forGeneric(...)`). This deserialization schema expects that
- the serialized records DO NOT contain embedded schema.
+3. `AvroDeserializationSchema` 使用静态提供的 schema 读取 Avro 格式的序列化数据。
+ 它能够从 Avro 生成的类（`AvroDeserializationSchema.forSpecific(...)`）中推断出 schema，或者可以与 `GenericRecords`
+ 一起使用手动提供的 schema（用 `AvroDeserializationSchema.forGeneric(...)`）。此反序列化 schema 要求序列化记录不能包含嵌入式架构！
- - There is also a version of this schema available that can lookup the writer's schema (schema which was used to write the record) in
- [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/docs/index.html). Using these deserialization schema
- record will be read with the schema that was retrieved from Schema Registry and transformed to a statically provided( either through
- `ConfluentRegistryAvroDeserializationSchema.forGeneric(...)` or `ConfluentRegistryAvroDeserializationSchema.forSpecific(...)`).
+ - 此模式还有一个版本，可以在 [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/docs/index.html) 中查找编写器的 schema（用于编写记录的 schema）。
+ - 使用这些反序列化 schema 记录将读取从 schema 注册表检索到的 schema 转换为静态提供的 schema（或者通过 `ConfluentRegistryAvroDeserializationSchema.forGeneric(...)` 或 `ConfluentRegistryAvroDeserializationSchema.forSpecific(...)`）。
- To use this deserialization schema one has to add the following additional dependency:
+ 要使用此反序列化 schema 必须添加以下依赖：
@@ -254,33 +207,24 @@ For convenience, Flink provides the following schemas:
-When encountering a corrupted message that cannot be deserialized for any reason, there
-are two options - either throwing an exception from the `deserialize(...)` method
-which will cause the job to fail and be restarted, or returning `null` to allow
-the Flink Kafka consumer to silently skip the corrupted message. Note that
-due to the consumer's fault tolerance (see below sections for more details),
-failing the job on the corrupted message will let the consumer attempt
-to deserialize the message again. Therefore, if deserialization still fails, the
-consumer will fall into a non-stop restart and fail loop on that corrupted
-message.
+当遇到因一些原因而无法反序列化的损坏消息时，这里有两个选项 - 从 `deserialize（...）` 方法抛出异常会导致作业失败并重新启动，或返回 `null`，以允许 Flink Kafka 消费者悄悄地跳过损坏的消息。请注意，由于 Consumer 的容错能力（请参阅下面的部分以获取更多详细信息），在损坏的消息上失败作业将使 consumer 尝试再次反序列化消息。因此，如果反序列化仍然失败，则 consumer 将在该损坏的消息上进入不间断重启和失败的循环。
-### Kafka Consumers Start Position Configuration
+### 配置 Kafka Consumer 开始消费的位置
-The Flink Kafka Consumer allows configuring how the start position for Kafka
-partitions are determined.
+Flink Kafka Consumer 允许通过配置来确定 Kafka 分区的起始位置。
-Example:
+例如：
{% highlight java %}
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-FlinkKafkaConsumer08 myConsumer = new FlinkKafkaConsumer08<>(...);
-myConsumer.setStartFromEarliest(); // start from the earliest record possible
-myConsumer.setStartFromLatest(); // start from the latest record
-myConsumer.setStartFromTimestamp(...); // start from specified epoch timestamp (milliseconds)
-myConsumer.setStartFromGroupOffsets(); // the default behaviour
+FlinkKafkaConsumer010 myConsumer = new FlinkKafkaConsumer010<>(...);
+myConsumer.setStartFromEarliest(); // 尽可能从最早的记录开始
+myConsumer.setStartFromLatest(); // 从最新的记录开始
+myConsumer.setStartFromTimestamp(...); // 从指定的时间开始（毫秒）
+myConsumer.setStartFromGroupOffsets(); // 默认的方法
DataStream stream = env.addSource(myConsumer);
...
@@ -290,11 +234,11 @@ DataStream stream = env.addSource(myConsumer);
{% highlight scala %}
val env = StreamExecutionEnvironment.getExecutionEnvironment()
-val myConsumer = new FlinkKafkaConsumer08[String](...)
-myConsumer.setStartFromEarliest() // start from the earliest record possible
-myConsumer.setStartFromLatest() // start from the latest record
-myConsumer.setStartFromTimestamp(...) // start from specified epoch timestamp (milliseconds)
-myConsumer.setStartFromGroupOffsets() // the default behaviour
+val myConsumer = new FlinkKafkaConsumer010[String](...)
+myConsumer.setStartFromEarliest() // 尽可能从最早的记录开始
+myConsumer.setStartFromLatest() // 从最新的记录开始
+myConsumer.setStartFromTimestamp(...) // 从指定的时间开始（毫秒）
+myConsumer.setStartFromGroupOffsets() // 默认的方法
val stream = env.addSource(myConsumer)
...
@@ -302,22 +246,14 @@ val stream = env.addSource(myConsumer)
-All versions of the Flink Kafka Consumer have the above explicit configuration methods for start position.
+Flink Kafka Consumer 的所有版本都具有上述明确的起始位置配置方法。
- * `setStartFromGroupOffsets` (default behaviour): Start reading partitions from
- the consumer group's (`group.id` setting in the consumer properties) committed
- offsets in Kafka brokers (or Zookeeper for Kafka 0.8). If offsets could not be
- found for a partition, the `auto.offset.reset` setting in the properties will be used.
- * `setStartFromEarliest()` / `setStartFromLatest()`: Start from the earliest / latest
- record. Under these modes, committed offsets in Kafka will be ignored and
- not used as starting positions.
- * `setStartFromTimestamp(long)`: Start from the specified timestamp. For each partition, the record
- whose timestamp is larger than or equal to the specified timestamp will be used as the start position.
- If a partition's latest record is earlier than the timestamp, the partition will simply be read
- from the latest record. Under this mode, committed offsets in Kafka will be ignored and not used as
- starting positions.
+ * `setStartFromGroupOffsets`（默认方法）：从 Kafka brokers 中的 consumer 组（consumer 属性中的 `group.id` 设置）提交的偏移量中开始读取分区。
+ 如果找不到分区的偏移量，那么将会使用配置中的 `auto.offset.reset` 设置。
+ * `setStartFromEarliest()` 或者 `setStartFromLatest()`：从最早或者最新的记录开始消费，在这些模式下，Kafka 中的 committed offset 将被忽略，不会用作起始位置。
+ * `setStartFromTimestamp(long)`：从指定的时间戳开始。对于每个分区，其时间戳大于或等于指定时间戳的记录将用作起始位置。如果一个分区的最新记录早于指定的时间戳，则只从最新记录读取该分区数据。在这种模式下，Kafka 中的已提交 offset 将被忽略，不会用作起始位置。
-You can also specify the exact offsets the consumer should start from for each partition:
+你也可以为每个分区指定 consumer 应该开始消费的具体 offset：
-The above example configures the consumer to start from the specified offsets for
-partitions 0, 1, and 2 of topic `myTopic`. The offset values should be the
-next record that the consumer should read for each partition. Note that
-if the consumer needs to read a partition which does not have a specified
-offset within the provided offsets map, it will fallback to the default
-group offsets behaviour (i.e. `setStartFromGroupOffsets()`) for that
-particular partition.
+上面的例子中使用的配置是指定从 `myTopic` 主题的 0 、1 和 2 分区的指定偏移量开始消费。offset 值是 consumer 应该为每个分区读取的下一条消息。请注意：如果 consumer 需要读取在提供的 offset 映射中没有指定 offset 的分区，那么它将回退到该特定分区的默认组偏移行为（即 `setStartFromGroupOffsets()`）。
+
-Note that these start position configuration methods do not affect the start position when the job is
-automatically restored from a failure or manually restored using a savepoint.
-On restore, the start position of each Kafka partition is determined by the
-offsets stored in the savepoint or checkpoint
-(please see the next section for information about checkpointing to enable
-fault tolerance for the consumer).
+请注意：当 Job 从故障中自动恢复或使用 savepoint 手动恢复时，这些起始位置配置方法不会影响消费的起始位置。在恢复时，每个 Kafka 分区的起始位置由存储在 savepoint 或 checkpoint 中的 offset 确定（有关 checkpointing 的信息，请参阅下一节，以便为 consumer 启用容错功能）。
-### Kafka Consumers and Fault Tolerance
+### Kafka Consumer 和容错
-With Flink's checkpointing enabled, the Flink Kafka Consumer will consume records from a topic and periodically checkpoint all
-its Kafka offsets, together with the state of other operations, in a consistent manner. In case of a job failure, Flink will restore
-the streaming program to the state of the latest checkpoint and re-consume the records from Kafka, starting from the offsets that were
-stored in the checkpoint.
+伴随着启用 Flink 的 checkpointing 后，Flink Kafka Consumer 将使用 topic 中的记录，并以一致的方式定期检查其所有 Kafka offset 和其他算子的状态。如果 Job 失败，Flink 会将流式程序恢复到最新 checkpoint 的状态，并从存储在 checkpoint 中的 offset 开始重新消费 Kafka 中的消息。
-The interval of drawing checkpoints therefore defines how much the program may have to go back at most, in case of a failure.
+因此，设置 checkpoint 的间隔定义了程序在发生故障时最多需要返回多少。
-To use fault tolerant Kafka Consumers, checkpointing of the topology needs to be enabled at the execution environment:
+要使用容错的 Kafka Consumer，需要在执行环境中启用拓扑的 checkpointing。
-Also note that Flink can only restart the topology if enough processing slots are available to restart the topology.
-So if the topology fails due to loss of a TaskManager, there must still be enough slots available afterwards.
-Flink on YARN supports automatic restart of lost YARN containers.
+另请注意，只有当可用的 slots 足够时，Flink 才能重新启动。因此，如果拓扑由于丢失了 TaskManager 而失败，那么之后必须要一直有足够可用的 solt。Flink on YARN 支持自动重启丢失的 YARN 容器。
-If checkpointing is not enabled, the Kafka consumer will periodically commit the offsets to Zookeeper.
+如果未启用 checkpoint，那么 Kafka consumer 将定期向 Zookeeper 提交 offset。
-### Kafka Consumers Topic and Partition Discovery
+### Kafka Consumer Topic 和分区发现
-#### Partition discovery
+#### 分区发现
-The Flink Kafka Consumer supports discovering dynamically created Kafka partitions, and consumes them with
-exactly-once guarantees. All partitions discovered after the initial retrieval of partition metadata (i.e., when the
-job starts running) will be consumed from the earliest possible offset.
+Flink Kafka Consumer 支持发现动态创建的 Kafka 分区，并使用精准一次的语义保证去消耗它们。在初始检索分区元数据之后（即，当 Job 开始运行时）发现的所有分区将从最早可能的 offset 中消费。
-By default, partition discovery is disabled. To enable it, set a non-negative value
-for `flink.partition-discovery.interval-millis` in the provided properties config,
-representing the discovery interval in milliseconds.
+默认情况下，是禁用了分区发现的。若要启用它，请在提供的属性配置中为 `flink.partition-discovery.interval-millis` 设置大于 0 的值，表示发现分区的间隔是以毫秒为单位的。
-Limitation When the consumer is restored from a savepoint from Flink versions
-prior to Flink 1.3.x, partition discovery cannot be enabled on the restore run. If enabled, the restore would fail
-with an exception. In this case, in order to use partition discovery, please first take a savepoint in Flink 1.3.x and
-then restore again from that.
+局限性 当从 Flink 1.3.x 之前的 Flink 版本的 savepoint 恢复 consumer 时，分区发现无法在恢复运行时启用。如果启用了，那么还原将会失败并且出现异常。在这种情况下，为了使用分区发现，请首先在 Flink 1.3.x 中使用 savepoint，然后再从 savepoint 中恢复。
-#### Topic discovery
+#### Topic 发现
-At a higher-level, the Flink Kafka Consumer is also capable of discovering topics, based on pattern matching on the
-topic names using regular expressions. See the below for an example:
+在更高的级别上，Flink Kafka Consumer 还能够使用正则表达式基于 Topic 名称的模式匹配来发现 Topic。请看下面的例子：
@@ -437,7 +350,7 @@ val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
properties.setProperty("group.id", "test")
-val myConsumer = new FlinkKafkaConsumer08[String](
+val myConsumer = new FlinkKafkaConsumer010[String](
java.util.regex.Pattern.compile("test-topic-[0-9]"),
new SimpleStringSchema,
properties)
@@ -448,82 +361,53 @@ val stream = env.addSource(myConsumer)
-In the above example, all topics with names that match the specified regular expression
-(starting with `test-topic-` and ending with a single digit) will be subscribed by the consumer
-when the job starts running.
-
-To allow the consumer to discover dynamically created topics after the job started running,
-set a non-negative value for `flink.partition-discovery.interval-millis`. This allows
-the consumer to discover partitions of new topics with names that also match the specified
-pattern.
-
-### Kafka Consumers Offset Committing Behaviour Configuration
-
-The Flink Kafka Consumer allows configuring the behaviour of how offsets
-are committed back to Kafka brokers (or Zookeeper in 0.8). Note that the
-Flink Kafka Consumer does not rely on the committed offsets for fault
-tolerance guarantees. The committed offsets are only a means to expose
-the consumer's progress for monitoring purposes.
-
-The way to configure offset commit behaviour is different, depending on
-whether or not checkpointing is enabled for the job.
-
- - *Checkpointing disabled:* if checkpointing is disabled, the Flink Kafka
- Consumer relies on the automatic periodic offset committing capability
- of the internally used Kafka clients. Therefore, to disable or enable offset
- committing, simply set the `enable.auto.commit` (or `auto.commit.enable`
- for Kafka 0.8) / `auto.commit.interval.ms` keys to appropriate values
- in the provided `Properties` configuration.
-
- - *Checkpointing enabled:* if checkpointing is enabled, the Flink Kafka
- Consumer will commit the offsets stored in the checkpointed states when
- the checkpoints are completed. This ensures that the committed offsets
- in Kafka brokers is consistent with the offsets in the checkpointed states.
- Users can choose to disable or enable offset committing by calling the
- `setCommitOffsetsOnCheckpoints(boolean)` method on the consumer (by default,
- the behaviour is `true`).
- Note that in this scenario, the automatic periodic offset committing
- settings in `Properties` is completely ignored.
-
-### Kafka Consumers and Timestamp Extraction/Watermark Emission
-
-In many scenarios, the timestamp of a record is embedded (explicitly or implicitly) in the record itself.
-In addition, the user may want to emit watermarks either periodically, or in an irregular fashion, e.g. based on
-special records in the Kafka stream that contain the current event-time watermark. For these cases, the Flink Kafka
-Consumer allows the specification of an `AssignerWithPeriodicWatermarks` or an `AssignerWithPunctuatedWatermarks`.
-
-You can specify your custom timestamp extractor/watermark emitter as described
-[here]({{ site.baseurl }}/dev/event_timestamps_watermarks.html), or use one from the
-[predefined ones]({{ site.baseurl }}/dev/event_timestamp_extractors.html). After doing so, you
-can pass it to your consumer in the following way:
+在上面的例子中，当 Job 开始运行时，Consumer 将订阅名称与指定正则表达式匹配的所有主题（以 `test-topic` 开头并以单个数字结尾）。
+
+要允许 consumer 在作业开始运行后发现动态创建的主题，那么请为 `flink.partition-discovery.interval-millis` 设置非负值。这允许 consumer 发现名称与指定模式匹配的新主题的分区。
+
+### Kafka Consumer 提交 Offset 的行为配置
+
+Flink Kafka Consumer 允许有配置如何将 offset 提交回 Kafka broker 的行为。请注意：Flink Kafka Consumer 不依赖于提交的 offset 来实现容错保证。提交的 offset 只是一种方法，用于公开 consumer 的进度以便进行监控。
+
+配置 offset 提交行为的方法是否相同，取决于是否为 job 启用了 checkpointing。
+
+ - *禁用 Checkpointing：* 如果禁用了 checkpointing，则 Flink Kafka Consumer 依赖于内部使用的 Kafka client 自动定期 offset 提交功能。
+ 因此，要禁用或启用 offset 的提交，只需将 `enable.auto.commit` 或者 `auto.commit.interval.ms` 的Key 值设置为提供的 `Properties` 配置中的适当值。
+
+ - *启用 Checkpointing：* 如果启用了 checkpointing，那么当 checkpointing 完成时，Flink Kafka Consumer 将提交的 offset 存储在 checkpoint 状态中。
+ 这确保 Kafka broker 中提交的 offset 与 checkpoint 状态中的 offset 一致。
+ 用户可以通过调用 consumer 上的 `setCommitOffsetsOnCheckpoints(boolean)` 方法来禁用或启用 offset 的提交(默认情况下，这个值是 true )。
+ 注意，在这个场景中，`Properties` 中的自动定期 offset 提交设置会被完全忽略。
+
+### Kafka Consumer 和 时间戳抽取以及 watermark 发送
+
+在许多场景中，记录的时间戳是(显式或隐式)嵌入到记录本身中。此外，用户可能希望定期或以不规则的方式 Watermark，例如基于 Kafka 流中包含当前事件时间的 watermark 的特殊记录。对于这些情况，Flink Kafka Consumer 允许指定 `AssignerWithPeriodicWatermarks` 或 `AssignerWithPunctuatedWatermarks`。
+
+你可以按照[此处]({{ site.baseurl }}/zh/dev/event_timestamps_watermarks.html)的说明指定自定义时间戳抽取器或者 Watermark 发送器，或者使用 [内置的]({{ site.baseurl }}/zh/dev/event_timestamp_extractors.html)。你也可以通过以下方式将其传递给你的 consumer：
{% highlight java %}
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181");
properties.setProperty("group.id", "test");
-FlinkKafkaConsumer08 myConsumer =
- new FlinkKafkaConsumer08<>("topic", new SimpleStringSchema(), properties);
+FlinkKafkaConsumer010 myConsumer =
+ new FlinkKafkaConsumer010<>("topic", new SimpleStringSchema(), properties);
myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter());
DataStream stream = env
- .addSource(myConsumer)
- .print();
+ .addSource(myConsumer)
+ .print();
{% endhighlight %}
{% highlight scala %}
val properties = new Properties()
properties.setProperty("bootstrap.servers", "localhost:9092")
-// only required for Kafka 0.8
-properties.setProperty("zookeeper.connect", "localhost:2181")
properties.setProperty("group.id", "test")
-val myConsumer = new FlinkKafkaConsumer08[String]("topic", new SimpleStringSchema(), properties)
+val myConsumer = new FlinkKafkaConsumer010[String]("topic", new SimpleStringSchema(), properties)
myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter())
stream = env
.addSource(myConsumer)
@@ -532,28 +416,15 @@ stream = env
-Internally, an instance of the assigner is executed per Kafka partition.
-When such an assigner is specified, for each record read from Kafka, the
-`extractTimestamp(T element, long previousElementTimestamp)` is called to assign a timestamp to the record and
-the `Watermark getCurrentWatermark()` (for periodic) or the
-`Watermark checkAndGetNextWatermark(T lastElement, long extractedTimestamp)` (for punctuated) is called to determine
-if a new watermark should be emitted and with which timestamp.
-
-**Note**: If a watermark assigner depends on records read from Kafka to advance its watermarks
-(which is commonly the case), all topics and partitions need to have a continuous stream of records.
-Otherwise, the watermarks of the whole application cannot advance and all time-based operations,
-such as time windows or functions with timers, cannot make progress. A single idle Kafka partition causes this behavior.
-A Flink improvement is planned to prevent this from happening
-(see [FLINK-5479: Per-partition watermarks in FlinkKafkaConsumer should consider idle partitions](
-https://issues.apache.org/jira/browse/FLINK-5479)).
-In the meanwhile, a possible workaround is to send *heartbeat messages* to all consumed partitions that advance the watermarks of idle partitions.
+在内部，每个 Kafka 分区执行一个 assigner 实例。当指定了这样的 assigner 时，对于从 Kafka 读取的每条消息，调用 `extractTimestamp(T element, long previousElementTimestamp)` 来为记录分配时间戳，并为 `Watermark getCurrentWatermark()`（定期形式）或 `Watermark checkAndGetNextWatermark(T lastElement, long extractedTimestamp)`（打点形式）以确定是否应该发出新的 watermark 以及使用哪个时间戳。
+
+**请注意**：如果 watermark assigner 依赖于从 Kafka 读取的消息来上涨其 watermark (通常就是这种情况)，那么所有主题和分区都需要有连续的消息流。否则，整个应用程序的 watermark 将无法上涨，所有基于时间的算子(例如时间窗口或带有计时器的函数)也无法运行。单个的 Kafka 分区也会导致这种反应。这是一个已在计划中的 Flink 改进，目的是为了防止这种情况发生（请见[FLINK-5479: Per-partition watermarks in FlinkKafkaConsumer should consider idle partitions](https://issues.apache.org/jira/browse/FLINK-5479)）。同时，可能的解决方法是将*心跳消息*发送到所有 consumer 的分区里，从而上涨空闲分区的 watermark。
## Kafka Producer
-Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc. or just `FlinkKafkaProducer` for Kafka >= 1.0.0 versions).
-It allows writing a stream of records to one or more Kafka topics.
+Flink Kafka Producer 被称为 `FlinkKafkaProducer011`（或适用于 Kafka 0.10.0.x 版本的 `FlinkKafkaProducer010`，或适用于 Kafka >= 1.0.0 版本的 `FlinkKafkaProducer`）。它允许将消息流写入一个或多个 Kafka topic。
-Example:
+示例：
@@ -561,12 +432,12 @@ Example:
DataStream stream = ...;
FlinkKafkaProducer011 myProducer = new FlinkKafkaProducer011(
- "localhost:9092", // broker list
- "my-topic", // target topic
- new SimpleStringSchema()); // serialization schema
+ "localhost:9092", // broker 列表
+ "my-topic", // 目标 topic
+ new SimpleStringSchema()); // 序列化 schema
-// versions 0.10+ allow attaching the records' event timestamp when writing them to Kafka;
-// this method is not available for earlier Kafka versions
+// 0.10+ 版本的 Kafka 允许在将记录写入 Kafka 时附加记录的事件时间戳；
+// 此方法不适用于早期版本的 Kafka
myProducer.setWriteTimestampToKafka(true);
stream.addSink(myProducer);
@@ -577,12 +448,12 @@ stream.addSink(myProducer);
val stream: DataStream[String] = ...
val myProducer = new FlinkKafkaProducer011[String](
- "localhost:9092", // broker list
- "my-topic", // target topic
- new SimpleStringSchema) // serialization schema
+ "localhost:9092", // broker 列表
+ "my-topic", // 目标 topic
+ new SimpleStringSchema) // 序列化 schema
-// versions 0.10+ allow attaching the records' event timestamp when writing them to Kafka;
-// this method is not available for earlier Kafka versions
+// 0.10+ 版本的 Kafka 允许在将记录写入 Kafka 时附加记录的事件时间戳；
+// 此方法不适用于早期版本的 Kafka
myProducer.setWriteTimestampToKafka(true)
stream.addSink(myProducer)
@@ -590,158 +461,84 @@ stream.addSink(myProducer)
-The above examples demonstrate the basic usage of creating a Flink Kafka Producer
-to write streams to a single Kafka target topic. For more advanced usages, there
-are other constructor variants that allow providing the following:
-
- * *Providing custom properties*:
- The producer allows providing a custom properties configuration for the internal `KafkaProducer`.
- Please refer to the [Apache Kafka documentation](https://kafka.apache.org/documentation.html) for
- details on how to configure Kafka Producers.
- * *Custom partitioner*: To assign records to specific
- partitions, you can provide an implementation of a `FlinkKafkaPartitioner` to the
- constructor. This partitioner will be called for each record in the stream
- to determine which exact partition of the target topic the record should be sent to.
- Please see [Kafka Producer Partitioning Scheme](#kafka-producer-partitioning-scheme) for more details.
- * *Advanced serialization schema*: Similar to the consumer,
- the producer also allows using an advanced serialization schema called `KeyedSerializationSchema`,
- which allows serializing the key and value separately. It also allows to override the target topic,
- so that one producer instance can send data to multiple topics.
-
-### Kafka Producer Partitioning Scheme
-
-By default, if a custom partitioner is not specified for the Flink Kafka Producer, the producer will use
-a `FlinkFixedPartitioner` that maps each Flink Kafka Producer parallel subtask to a single Kafka partition
-(i.e., all records received by a sink subtask will end up in the same Kafka partition).
-
-A custom partitioner can be implemented by extending the `FlinkKafkaPartitioner` class. All
-Kafka versions' constructors allow providing a custom partitioner when instantiating the producer.
-Note that the partitioner implementation must be serializable, as they will be transferred across Flink nodes.
-Also, keep in mind that any state in the partitioner will be lost on job failures since the partitioner
-is not part of the producer's checkpointed state.
-
-It is also possible to completely avoid using and kind of partitioner, and simply let Kafka partition
-the written records by their attached key (as determined for each record using the provided serialization schema).
-To do this, provide a `null` custom partitioner when instantiating the producer. It is important
-to provide `null` as the custom partitioner; as explained above, if a custom partitioner is not specified
-the `FlinkFixedPartitioner` is used instead.
-
-### Kafka Producers and Fault Tolerance
-
-#### Kafka 0.8
-
-Before 0.9 Kafka did not provide any mechanisms to guarantee at-least-once or exactly-once semantics.
-
-#### Kafka 0.9 and 0.10
-
-With Flink's checkpointing enabled, the `FlinkKafkaProducer09` and `FlinkKafkaProducer010`
-can provide at-least-once delivery guarantees.
-
-Besides enabling Flink's checkpointing, you should also configure the setter
-methods `setLogFailuresOnly(boolean)` and `setFlushOnCheckpoint(boolean)` appropriately.
-
- * `setLogFailuresOnly(boolean)`: by default, this is set to `false`.
- Enabling this will let the producer only log failures
- instead of catching and rethrowing them. This essentially accounts the record
- to have succeeded, even if it was never written to the target Kafka topic. This
- must be disabled for at-least-once.
- * `setFlushOnCheckpoint(boolean)`: by default, this is set to `true`.
- With this enabled, Flink's checkpoints will wait for any
- on-the-fly records at the time of the checkpoint to be acknowledged by Kafka before
- succeeding the checkpoint. This ensures that all records before the checkpoint have
- been written to Kafka. This must be enabled for at-least-once.
-
-In conclusion, the Kafka producer by default has at-least-once guarantees for versions
-0.9 and 0.10, with `setLogFailureOnly` set to `false` and `setFlushOnCheckpoint` set
-to `true`.
-
-**Note**: By default, the number of retries is set to "0". This means that when `setLogFailuresOnly` is set to `false`,
-the producer fails immediately on errors, including leader changes. The value is set to "0" by default to avoid
-duplicate messages in the target topic that are caused by retries. For most production environments with frequent broker changes,
-we recommend setting the number of retries to a higher value.
-
-**Note**: There is currently no transactional producer for Kafka, so Flink can not guarantee exactly-once delivery
-into a Kafka topic.
-
-#### Kafka 0.11 and newer
-
-With Flink's checkpointing enabled, the `FlinkKafkaProducer011` (`FlinkKafkaProducer` for Kafka >= 1.0.0 versions) can provide
-exactly-once delivery guarantees.
-
-Besides enabling Flink's checkpointing, you can also choose three different modes of operating
-chosen by passing appropriate `semantic` parameter to the `FlinkKafkaProducer011` (`FlinkKafkaProducer` for Kafka >= 1.0.0 versions):
-
- * `Semantic.NONE`: Flink will not guarantee anything. Produced records can be lost or they can
- be duplicated.
- * `Semantic.AT_LEAST_ONCE` (default setting): similar to `setFlushOnCheckpoint(true)` in
- `FlinkKafkaProducer010`. This guarantees that no records will be lost (although they can be duplicated).
- * `Semantic.EXACTLY_ONCE`: uses Kafka transactions to provide exactly-once semantic. Whenever you write
- to Kafka using transactions, do not forget about setting desired `isolation.level` (`read_committed`
- or `read_uncommitted` - the latter one is the default value) for any application consuming records
- from Kafka.
-
-##### Caveats
-
-`Semantic.EXACTLY_ONCE` mode relies on the ability to commit transactions
-that were started before taking a checkpoint, after recovering from the said checkpoint. If the time
-between Flink application crash and completed restart is larger than Kafka's transaction timeout
-there will be data loss (Kafka will automatically abort transactions that exceeded timeout time).
-Having this in mind, please configure your transaction timeout appropriately to your expected down
-times.
-
-Kafka brokers by default have `transaction.max.timeout.ms` set to 15 minutes. This property will
-not allow to set transaction timeouts for the producers larger than it's value.
-`FlinkKafkaProducer011` by default sets the `transaction.timeout.ms` property in producer config to
-1 hour, thus `transaction.max.timeout.ms` should be increased before using the
-`Semantic.EXACTLY_ONCE` mode.
-
-In `read_committed` mode of `KafkaConsumer`, any transactions that were not finished
-(neither aborted nor completed) will block all reads from the given Kafka topic past any
-un-finished transaction. In other words after following sequence of events:
-
-1. User started `transaction1` and written some records using it
-2. User started `transaction2` and written some further records using it
-3. User committed `transaction2`
-
-Even if records from `transaction2` are already committed, they will not be visible to
-the consumers until `transaction1` is committed or aborted. This has two implications:
-
- * First of all, during normal working of Flink applications, user can expect a delay in visibility
- of the records produced into Kafka topics, equal to average time between completed checkpoints.
- * Secondly in case of Flink application failure, topics into which this application was writing,
- will be blocked for the readers until the application restarts or the configured transaction
- timeout time will pass. This remark only applies for the cases when there are multiple
- agents/applications writing to the same Kafka topic.
-
-**Note**: `Semantic.EXACTLY_ONCE` mode uses a fixed size pool of KafkaProducers
-per each `FlinkKafkaProducer011` instance. One of each of those producers is used per one
-checkpoint. If the number of concurrent checkpoints exceeds the pool size, `FlinkKafkaProducer011`
-will throw an exception and will fail the whole application. Please configure max pool size and max
-number of concurrent checkpoints accordingly.
-
-**Note**: `Semantic.EXACTLY_ONCE` takes all possible measures to not leave any lingering transactions
-that would block the consumers from reading from Kafka topic more then it is necessary. However in the
-event of failure of Flink application before first checkpoint, after restarting such application there
-is no information in the system about previous pool sizes. Thus it is unsafe to scale down Flink
-application before first checkpoint completes, by factor larger than `FlinkKafkaProducer011.SAFE_SCALE_DOWN_FACTOR`.
+上面的例子演示了创建 Flink Kafka Producer 来将流消息写入单个 Kafka 目标 topic 的基本用法。
+对于更高级的用法，这还有其他构造函数变体允许提供以下内容：
+
+ * *提供自定义属性*：producer 允许为内部 `KafkaProducer` 提供自定义属性配置。有关如何配置 Kafka Producer 的详细信息，请参阅 [Apache Kafka 文档](https://kafka.apache.org/documentation.html)。
+ * *自定义分区器*：要将消息分配给特定的分区，可以向构造函数提供一个 `FlinkKafkaPartitioner` 的实现。这个分区器将被流中的每条记录调用，以确定消息应该发送到目标 topic 的哪个具体分区里。有关详细信息，请参阅 [Kafka Producer 分区方案](#kafka-producer-分区方案)。
+ * *高级的序列化 schema*：与 consumer 类似，producer 还允许使用名为 `KeyedSerializationSchema` 的高级序列化 schema，该 schema 允许单独序列化 key 和 value。它还允许覆盖目标 topic，以便 producer 实例可以将数据发送到多个 topic。
+
+### Kafka Producer 分区方案
+
+默认情况下，如果没有为 Flink Kafka Producer 指定自定义分区程序，则 producer 将使用 `FlinkFixedPartitioner` 为每个 Flink Kafka Producer 并行子任务映射到单个 Kafka 分区（即，接收子任务接收到的所有消息都将位于同一个 Kafka 分区中）。
+
+可以通过扩展 `FlinkKafkaPartitioner` 类来实现自定义分区程序。所有 Kafka 版本的构造函数都允许在实例化 producer 时提供自定义分区程序。
+注意：分区器实现必须是可序列化的，因为它们将在 Flink 节点之间传输。此外，请记住分区器中的任何状态都将在作业失败时丢失，因为分区器不是 producer 的 checkpoint 状态的一部分。
+
+也可以完全避免使用分区器，并简单地让 Kafka 通过其附加 key 写入的消息进行分区（使用提供的序列化 schema 为每条记录确定分区）。
+为此，在实例化 producer 时提供 `null` 自定义分区程序，提供 `null` 作为自定义分区器是很重要的; 如上所述，如果未指定自定义分区程序，则默认使用 `FlinkFixedPartitioner`。
+
+### Kafka Producer 和容错
+
+#### Kafka 0.10
+
+启用 Flink 的 checkpointing 后，`FlinkKafkaProducer010` 可以提供至少一次的语义。
-## Using Kafka timestamps and Flink event time in Kafka 0.10
-
-Since Apache Kafka 0.10+, Kafka's messages can carry
-[timestamps](https://cwiki.apache.org/confluence/display/KAFKA/KIP-32+-+Add+timestamps+to+Kafka+message), indicating
-the time the event has occurred (see ["event time" in Apache Flink](../event_time.html)) or the time when the message
-has been written to the Kafka broker.
-
-The `FlinkKafkaConsumer010` will emit records with the timestamp attached, if the time characteristic in Flink is
-set to `TimeCharacteristic.EventTime` (`StreamExecutionEnvironment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)`).
-
-The Kafka consumer does not emit watermarks. To emit watermarks, the same mechanisms as described above in
-"Kafka Consumers and Timestamp Extraction/Watermark Emission" using the `assignTimestampsAndWatermarks` method are applicable.
-
-There is no need to define a timestamp extractor when using the timestamps from Kafka. The `previousElementTimestamp` argument of
-the `extractTimestamp()` method contains the timestamp carried by the Kafka message.
-
-A timestamp extractor for a Kafka consumer would look like this:
+除了启用 Flink 的 checkpointing 之外，还应该适当地配置 setter 方法，`setLogFailuresOnly(boolean)` 和 `setFlushOnCheckpoint(boolean)`。
+
+ * `setLogFailuresOnly(boolean)`：默认情况下，此值设置为 `false`。启用这个选项将使 producer 仅记录失败而不是捕获和重新抛出它们。这基本上是记录了成功的记录，即使它从未写入目标 Kafka topic。对 at-least-once 的语义，这个方法必须禁用。
+ * `setFlushOnCheckpoint(boolean)`：默认情况下，此值设置为 `true`。启用此功能后，Flink 的 checkpoint 将在 checkpoint 成功之前等待 Kafka 确认 checkpoint 时的任意即时记录。这样可确保 checkpoint 之前的所有记录都已写入 Kafka。对 at-least-once 的语义，这个方法必须启用。
+
+总之，默认情况下，Kafka producer 中，`setLogFailureOnly` 设置为 `false` 及 `setFlushOnCheckpoint` 设置为 `true` 会为 0.10 版本提供 at-least-once 语义。
+
+**注意**：默认情况下，重试次数设置为 0。这意味着当 `setLogFailuresOnly` 设置为 `false` 时，producer 会立即失败，包括 leader 更改。该值默认设置为 0，以避免重试导致目标 topic 中出现重复的消息。对于大多数频繁更改 broker 的生产环境，我们建议将重试次数设置为更高的值。
+
+**注意**：目前还没有 Kafka 的事务 producer，所以 Flink 不能保证写入 Kafka topic 的精准一次语义。
+
+#### Kafka 0.11 和更新的版本
+
+启用 Flink 的 checkpointing 后，`FlinkKafkaProducer011`（适用于 Kafka >= 1.0.0 版本的 `FlinkKafkaProducer`）可以提供精准一次的语义保证。
+
+除了启用 Flink 的 checkpointing，还可以通过将适当的 `semantic` 参数传递给 `FlinkKafkaProducer011`（适用于 Kafka >= 1.0.0 版本的 `FlinkKafkaProducer`）来选择三种不同的操作模式：
+
+ * `Semantic.NONE`：Flink 不会有任何语义的保证，产生的记录可能会丢失或重复。
+ * `Semantic.AT_LEAST_ONCE`（默认设置）：类似 `FlinkKafkaProducer010` 中的 `setFlushOnCheckpoint(true)`，这可以保证不会丢失任何记录（虽然记录可能会重复）。
+ * `Semantic.EXACTLY_ONCE`：使用 Kafka 事务提供精准一次的语义。无论何时，在使用事务写入 Kafka 时，都要记得为所有消费 Kafka 消息的应用程序设置所需的 `isolation.level`（ `read_committed` 或 `read_uncommitted` - 后者是默认值）。
+
+##### 注意事项
+
+`Semantic.EXACTLY_ONCE` 模式依赖于事务提交的能力。事务提交发生于触发 checkpoint 之前，以及从 checkpoint 恢复之后。如果从 Flink 应用程序崩溃到完全重启的时间超过了 Kafka 的事务超时时间，那么将会有数据丢失（Kafka 会自动丢弃超出超时时间的事务）。考虑到这一点，请根据预期的宕机时间来合理地配置事务超时时间。
+
+默认情况下，Kafka broker 将 `transaction.max.timeout.ms` 设置为 15 分钟。此属性不允许为大于其值的 producer 设置事务超时时间。
+默认情况下，`FlinkKafkaProducer011` 将 producer config 中的 `transaction.timeout.ms` 属性设置为 1 小时，因此在使用 `Semantic.EXACTLY_ONCE` 模式之前应该增加 `transaction.max.timeout.ms` 的值。
+
+在 `KafkaConsumer` 的 `read_committed` 模式中，任何未结束（既未中止也未完成）的事务将阻塞来自给定 Kafka topic 的未结束事务之后的所有读取数据。
+换句话说，在遵循如下一系列事件之后：
+
+1. 用户启动了 `transaction1` 并使用它写了一些记录
+2. 用户启动了 `transaction2` 并使用它编写了一些其他记录
+3. 用户提交了 `transaction2`
+
+即使 `transaction2` 中的记录已提交，在提交或中止 `transaction1` 之前，消费者也不会看到这些记录。这有 2 层含义：
+
+ * 首先，在 Flink 应用程序的正常工作期间，用户可以预料 Kafka 主题中生成的记录的可见性会延迟，相当于已完成 checkpoint 之间的平均时间。
+ * 其次，在 Flink 应用程序失败的情况下，此应用程序正在写入的供消费者读取的主题将被阻塞，直到应用程序重新启动或配置的事务超时时间过去后，才恢复正常。此标注仅适用于有多个 agent 或者应用程序写入同一 Kafka 主题的情况。
+
+**注意**：`Semantic.EXACTLY_ONCE` 模式为每个 `FlinkKafkaProducer011` 实例使用固定大小的 KafkaProducer 池。每个 checkpoint 使用其中一个 producer。如果并发 checkpoint 的数量超过池的大小，`FlinkKafkaProducer011` 将抛出异常，并导致整个应用程序失败。请合理地配置最大池大小和最大并发 checkpoint 数量。
+
+**注意**：`Semantic.EXACTLY_ONCE` 会尽一切可能不留下任何逗留的事务，否则会阻塞其他消费者从这个 Kafka topic 中读取数据。但是，如果 Flink 应用程序在第一次 checkpoint 之前就失败了，那么在重新启动此类应用程序后，系统中不会有先前池大小（pool size）相关的信息。因此，在第一次 checkpoint 完成前对 Flink 应用程序进行缩容，且并发数缩容倍数大于安全系数 `FlinkKafkaProducer011.SAFE_SCALE_DOWN_FACTOR` 的值的话，是不安全的。
+
+## 在 Kafka 0.10.x 中使用 Kafka 时间戳和 Flink 事件时间
+
+自 Apache Kafka 0.10+ 以来，Kafka 的消息可以携带[时间戳](https://cwiki.apache.org/confluence/display/KAFKA/KIP-32+-+Add+timestamps+to+Kafka+message)，指示事件发生的时间（请参阅 [Apache Flink 中的"事件时间"](../event_time.html)）或消息写入 Kafka broker 的时间。
+
+如果 Flink 中的时间特性设置为 `TimeCharacteristic.EventTime`（ `StreamExecutionEnvironment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)`），则 `FlinkKafkaConsumer010` 将发出附加时间戳的记录。
+
+Kafka consumer 不会发出 watermark。为了发出 watermark，可采用上文 “Kafka Consumer 和时间戳抽取以及 watermark 发送” 章节中描述的 `assignTimestampsAndWatermarks` 方法。
+
+使用 Kafka 的时间戳时，无需定义时间戳提取器。`extractTimestamp()` 方法的 `previousElementTimestamp` 参数包含 `Kafka` 消息携带的时间戳。
+
+Kafka consumer 的时间戳提取器应该是这样的：
{% highlight java %}
public long extractTimestamp(Long element, long previousElementTimestamp) {
return previousElementTimestamp;
@@ -750,7 +547,7 @@ public long extractTimestamp(Long element, long previousElementTimestamp) {
-The `FlinkKafkaProducer010` only emits the record timestamp, if `setWriteTimestampToKafka(true)` is set.
+只有设置了 `setWriteTimestampToKafka(true)`，则 `FlinkKafkaProducer010` 才会发出记录时间戳。
{% highlight java %}
FlinkKafkaProducer010.FlinkKafkaProducer010Configuration config = FlinkKafkaProducer010.writeToKafkaWithTimestamps(streamWithTimestamps, topic, new SimpleStringSchema(), standardProps);
@@ -759,85 +556,50 @@ config.setWriteTimestampToKafka(true);
-## Kafka Connector metrics
-
-Flink's Kafka connectors provide some metrics through Flink's [metrics system]({{ site.baseurl }}/monitoring/metrics.html) to analyze
-the behavior of the connector.
-The producers export Kafka's internal metrics through Flink's metric system for all supported versions. The consumers export
-all metrics starting from Kafka version 0.9. The Kafka documentation lists all exported metrics
-in its [documentation](http://kafka.apache.org/documentation/#selector_monitoring).
+## Kafka 连接器指标
-In addition to these metrics, all consumers expose the `current-offsets` and `committed-offsets` for each topic partition.
-The `current-offsets` refers to the current offset in the partition. This refers to the offset of the last element that
-we retrieved and emitted successfully. The `committed-offsets` is the last committed offset.
+Flink 的 Kafka 连接器通过 Flink 的 [metric 系统]({{ site.baseurl }}/zh/monitoring/metrics.html) 提供一些指标来分析 Kafka Connector 的状况。Producer 通过 Flink 的 metrics 系统为所有支持的版本导出 Kafka 的内部指标。consumer 从 Kafka 0.10 版本开始导出所有指标。Kafka 文档在其[文档](http://kafka.apache.org/documentation/#selector_monitoring)中列出了所有导出的指标。
-The Kafka Consumers in Flink commit the offsets back to Zookeeper (Kafka 0.8) or the Kafka brokers (Kafka 0.9+). If checkpointing
-is disabled, offsets are committed periodically.
-With checkpointing, the commit happens once all operators in the streaming topology have confirmed that they've created a checkpoint of their state.
-This provides users with at-least-once semantics for the offsets committed to Zookeeper or the broker. For offsets checkpointed to Flink, the system
-provides exactly once guarantees.
+除了这些指标之外，所有 consumer 都暴露了每个主题分区的 `current-offsets` 和 `committed-offsets`。`current-offsets` 是指分区中的当前偏移量。指的是我们成功检索和发出的最后一个元素的偏移量。`committed-offsets` 是最后提交的偏移量。这为用户提供了 at-least-once 语义，用于提交给 Zookeeper 或 broker 的偏移量。对于 Flink 的偏移检查点，系统提供精准一次语义。
-The offsets committed to ZK or the broker can also be used to track the read progress of the Kafka consumer. The difference between
-the committed offset and the most recent offset in each partition is called the *consumer lag*. If the Flink topology is consuming
-the data slower from the topic than new data is added, the lag will increase and the consumer will fall behind.
-For large production deployments we recommend monitoring that metric to avoid increasing latency.
+提交给 ZK 或 broker 的偏移量也可以用来跟踪 Kafka consumer 的读取进度。每个分区中提交的偏移量和最近偏移量之间的差异称为 *consumer lag*。如果 Flink 拓扑消耗来自 topic 的数据的速度比添加新数据的速度慢，那么延迟将会增加，consumer 将会滞后。对于大型生产部署，我们建议监视该指标，以避免增加延迟。
-## Enabling Kerberos Authentication (for versions 0.9+ and above only)
+## 启用 Kerberos 身份验证
-Flink provides first-class support through the Kafka connector to authenticate to a Kafka installation
-configured for Kerberos. Simply configure Flink in `flink-conf.yaml` to enable Kerberos authentication for Kafka like so:
+Flink 通过 Kafka 连接器提供了一流的支持，可以对 Kerberos 配置的 Kafka 安装进行身份验证。只需在 `flink-conf.yaml` 中配置 Flink。像这样为 Kafka 启用 Kerberos 身份验证：
-1. Configure Kerberos credentials by setting the following -
- - `security.kerberos.login.use-ticket-cache`: By default, this is `true` and Flink will attempt to use Kerberos credentials in ticket caches managed by `kinit`.
- Note that when using the Kafka connector in Flink jobs deployed on YARN, Kerberos authorization using ticket caches will not work.
- This is also the case when deploying using Mesos, as authorization using ticket cache is not supported for Mesos deployments.
- - `security.kerberos.login.keytab` and `security.kerberos.login.principal`: To use Kerberos keytabs instead, set values for both of these properties.
+1. 通过设置以下内容配置 Kerberos 票据
+ - `security.kerberos.login.use-ticket-cache`：默认情况下，这个值是 `true`，Flink 将尝试在 `kinit` 管理的票据缓存中使用 Kerberos 票据。注意！在 YARN 上部署的 Flink jobs 中使用 Kafka 连接器时，使用票据缓存的 Kerberos 授权将不起作用。使用 Mesos 进行部署时也是如此，因为 Mesos 部署不支持使用票据缓存进行授权。
+ - `security.kerberos.login.keytab` 和 `security.kerberos.login.principal`：要使用 Kerberos keytabs，需为这两个属性设置值。
-2. Append `KafkaClient` to `security.kerberos.login.contexts`: This tells Flink to provide the configured Kerberos credentials to the Kafka login context to be used for Kafka authentication.
+2. 将 `KafkaClient` 追加到 `security.kerberos.login.contexts`：这告诉 Flink 将配置的 Kerberos 票据提供给 Kafka 登录上下文以用于 Kafka 身份验证。
-Once Kerberos-based Flink security is enabled, you can authenticate to Kafka with either the Flink Kafka Consumer or Producer
-by simply including the following two settings in the provided properties configuration that is passed to the internal Kafka client:
+一旦启用了基于 Kerberos 的 Flink 安全性后，只需在提供的属性配置中包含以下两个设置（通过传递给内部 Kafka 客户端），即可使用 Flink Kafka Consumer 或 Producer 向 Kafk a进行身份验证：
-- Set `security.protocol` to `SASL_PLAINTEXT` (default `NONE`): The protocol used to communicate to Kafka brokers.
-When using standalone Flink deployment, you can also use `SASL_SSL`; please see how to configure the Kafka client for SSL [here](https://kafka.apache.org/documentation/#security_configclients).
-- Set `sasl.kerberos.service.name` to `kafka` (default `kafka`): The value for this should match the `sasl.kerberos.service.name` used for Kafka broker configurations.
-A mismatch in service name between client and server configuration will cause the authentication to fail.
+- 将 `security.protocol` 设置为 `SASL_PLAINTEXT`（默认为 `NONE`）：用于与 Kafka broker 进行通信的协议。使用独立 Flink 部署时，也可以使用 `SASL_SSL`;请在[此处](https://kafka.apache.org/documentation/#security_configclients)查看如何为 SSL 配置 Kafka 客户端。
+- 将 `sasl.kerberos.service.name` 设置为 `kafka`（默认为 `kafka`）：此值应与用于 Kafka broker 配置的 `sasl.kerberos.service.name` 相匹配。客户端和服务器配置之间的服务名称不匹配将导致身份验证失败。
-For more information on Flink configuration for Kerberos security, please see [here]({{ site.baseurl}}/ops/config.html).
-You can also find [here]({{ site.baseurl}}/ops/security-kerberos.html) further details on how Flink internally setups Kerberos-based security.
+有关 Kerberos 安全性 Flink 配置的更多信息，请参见[这里]({{ site.baseurl }}/zh/ops/config.html)。你也可以在[这里]({{ site.baseurl }}/zh/ops/security-kerberos.html)进一步了解 Flink 如何在内部设置基于 kerberos 的安全性。
-## Troubleshooting
+## 问题排查
-If you have a problem with Kafka when using Flink, keep in mind that Flink only wraps
-KafkaConsumer or
-KafkaProducer
-and your problem might be independent of Flink and sometimes can be solved by upgrading Kafka brokers,
-reconfiguring Kafka brokers or reconfiguring KafkaConsumer or KafkaProducer in Flink.
-Some examples of common problems are listed below.
+如果你在使用 Flink 时对 Kafka 有问题，请记住，Flink 只封装 KafkaConsumer 或 KafkaProducer，你的问题可能独立于 Flink，有时可以通过升级 Kafka broker 程序、重新配置 Kafka broker 程序或在 Flink 中重新配置 KafkaConsumer 或 KafkaProducer 来解决。下面列出了一些常见问题的示例。
-### Data loss
+### 数据丢失
-Depending on your Kafka configuration, even after Kafka acknowledges
-writes you can still experience data loss. In particular keep in mind about the following properties
-in Kafka config:
+根据你的 Kafka 配置，即使在 Kafka 确认写入后，你仍然可能会遇到数据丢失。特别要记住在 Kafka 的配置中设置以下属性：
- `acks`
- `log.flush.interval.messages`
- `log.flush.interval.ms`
- `log.flush.*`
-Default values for the above options can easily lead to data loss.
-Please refer to the Kafka documentation for more explanation.
+上述选项的默认值是很容易导致数据丢失的。请参考 Kafka 文档以获得更多的解释。
### UnknownTopicOrPartitionException
-One possible cause of this error is when a new leader election is taking place,
-for example after or during restarting a Kafka broker.
-This is a retriable exception, so Flink job should be able to restart and resume normal operation.
-It also can be circumvented by changing `retries` property in the producer settings.
-However this might cause reordering of messages,
-which in turn if undesired can be circumvented by setting `max.in.flight.requests.per.connection` to 1.
+导致此错误的一个可能原因是正在进行新的 leader 选举，例如在重新启动 Kafka broker 之后或期间。这是一个可重试的异常，因此 Flink job 应该能够重启并恢复正常运行。也可以通过更改 producer 设置中的 `retries` 属性来规避。但是，这可能会导致重新排序消息，反过来可以通过将 `max.in.flight.requests.per.connection` 设置为 1 来避免不需要的消息。
{% top %}
diff --git a/docs/dev/connectors/kinesis.md b/docs/dev/connectors/kinesis.md
index e23572586f6cf291e17cfbf785510d9fac182b94..ab4b76b1193867557e25c5a8dd0b98481910846b 100644
--- a/docs/dev/connectors/kinesis.md
+++ b/docs/dev/connectors/kinesis.md
@@ -414,17 +414,14 @@ starting point. If the queue size limits throughput (below 1MB per second per
shard), try increasing the queue limit slightly.
-## Using Non-AWS Kinesis Endpoints for Testing
+## Using Custom Kinesis Endpoints
-It is sometimes desirable to have Flink operate as a consumer or producer against a non-AWS Kinesis endpoint such as
-[Kinesalite](https://github.com/mhart/kinesalite); this is especially useful when performing functional testing of a Flink
-application. The AWS endpoint that would normally be inferred by the AWS region set in the Flink configuration must be overridden via a configuration property.
+It is sometimes desirable to have Flink operate as a consumer or producer against a Kinesis VPC endpoint or a non-AWS
+Kinesis endpoint such as [Kinesalite](https://github.com/mhart/kinesalite); this is especially useful when performing
+functional testing of a Flink application. The AWS endpoint that would normally be inferred by the AWS region set in the
+Flink configuration must be overridden via a configuration property.
-To override the AWS endpoint, taking the producer for example, set the `AWSConfigConstants.AWS_ENDPOINT` property in the
-Flink configuration, in addition to the `AWSConfigConstants.AWS_REGION` required by Flink. Although the region is
-required, it will not be used to determine the AWS endpoint URL.
-
-The following example shows how one might supply the `AWSConfigConstants.AWS_ENDPOINT` configuration property:
+To override the AWS endpoint, set the `AWSConfigConstants.AWS_ENDPOINT` and `AWSConfigConstants.AWS_REGION` properties. The region will be used to sign the endpoint URL.
diff --git a/docs/dev/connectors/kinesis.zh.md b/docs/dev/connectors/kinesis.zh.md
index 48849c912cac648c2532dfe55cb5a0d52e4cfb27..e7fd602efaca1d3b5ff37d3fdaee86f82eb708ae 100644
--- a/docs/dev/connectors/kinesis.zh.md
+++ b/docs/dev/connectors/kinesis.zh.md
@@ -414,17 +414,14 @@ starting point. If the queue size limits throughput (below 1MB per second per
shard), try increasing the queue limit slightly.
-## Using Non-AWS Kinesis Endpoints for Testing
+## Using Custom Kinesis Endpoints
-It is sometimes desirable to have Flink operate as a consumer or producer against a non-AWS Kinesis endpoint such as
-[Kinesalite](https://github.com/mhart/kinesalite); this is especially useful when performing functional testing of a Flink
-application. The AWS endpoint that would normally be inferred by the AWS region set in the Flink configuration must be overridden via a configuration property.
+It is sometimes desirable to have Flink operate as a consumer or producer against a Kinesis VPC endpoint or a non-AWS
+Kinesis endpoint such as [Kinesalite](https://github.com/mhart/kinesalite); this is especially useful when performing
+functional testing of a Flink application. The AWS endpoint that would normally be inferred by the AWS region set in the
+Flink configuration must be overridden via a configuration property.
-To override the AWS endpoint, taking the producer for example, set the `AWSConfigConstants.AWS_ENDPOINT` property in the
-Flink configuration, in addition to the `AWSConfigConstants.AWS_REGION` required by Flink. Although the region is
-required, it will not be used to determine the AWS endpoint URL.
-
-The following example shows how one might supply the `AWSConfigConstants.AWS_ENDPOINT` configuration property:
+To override the AWS endpoint, set the `AWSConfigConstants.AWS_ENDPOINT` and `AWSConfigConstants.AWS_REGION` properties. The region will be used to sign the endpoint URL.
diff --git a/docs/dev/connectors/streamfile_sink.md b/docs/dev/connectors/streamfile_sink.md
index c59e646a354898301df1aae32af0377cdc852071..bf25182179ee04bc108283e8b4022c193c1b3fc0 100644
--- a/docs/dev/connectors/streamfile_sink.md
+++ b/docs/dev/connectors/streamfile_sink.md
@@ -29,14 +29,14 @@ under the License.
This connector provides a Sink that writes partitioned files to filesystems
supported by the [Flink `FileSystem` abstraction]({{ site.baseurl}}/ops/filesystems/index.html).
-In order to handle unbounded data streams, the streaming file sink writes incoming data
-into buckets. The bucketing behaviour is fully configurable with a default time-based
-bucketing where we start writing a new bucket every hour and thus get files that correspond to
-records received during certain time intervals from the stream.
+The streaming file sink writes incoming data into buckets. Given that the incoming streams can be unbounded,
+data in each bucket are organized into part files of finite size. The bucketing behaviour is fully configurable
+with a default time-based bucketing where we start writing a new bucket every hour. This means that each resulting
+bucket will contain files with records received during 1 hour intervals from the stream.
-The bucket directories themselves contain several part files with the actual output data, with at least
-one for each subtask of the sink that has received data for the bucket. Additional part files will be created according to the configurable
-rolling policy. The default policy rolls files based on size, a timeout that specifies the maximum duration for which a file can be open, and a maximum inactivity timeout after which the file is closed.
+Data within the bucket directories are split into part files. Each bucket will contain at least one part file for
+each subtask of the sink that has received data for that bucket. Additional part files will be created according to the configurable
+rolling policy. The default policy rolls part files based on size, a timeout that specifies the maximum duration for which a file can be open, and a maximum inactivity timeout after which the file is closed.
IMPORTANT: Checkpointing needs to be enabled when using the StreamingFileSink. Part files can only be finalized
@@ -46,92 +46,6 @@ rolling policy. The default policy rolls files based on size, a timeout that spe
-### Bucket Assignment
-
-The bucketing logic defines how the data will be structured into subdirectories inside the base output directory.
-
-Both row and bulk formats use the [DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) as the default assigner.
-By default the DateTimeBucketAssigner creates hourly buckets based on the system default timezone
-with the following format: `yyyy-MM-dd--HH`. Both the date format (i.e. bucket size) and timezone can be
-configured manually.
-
-We can specify a custom [BucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssigner.html) by calling `.withBucketAssigner(assigner)` on the format builders.
-
-Flink comes with two built in BucketAssigners:
-
- - [DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) : Default time based assigner
- - [BasePathBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/BasePathBucketAssigner.html) : Assigner that stores all part files in the base path (single global bucket)
-
-### Rolling Policy
-
-The [RollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicy.html) defines when a given in-progress part file will be closed and moved to the pending and later to finished state.
-In combination with the checkpointing interval (pending files become finished on the next checkpoint) this controls how quickly
-part files become available for downstream readers and also the size and number of these parts.
-
-Flink comes with two built-in RollingPolicies:
-
- - [DefaultRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/DefaultRollingPolicy.html)
- - [OnCheckpointRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/OnCheckpointRollingPolicy.html)
-
-### Part file lifecycle
-
-In order to use the output of the StreamingFileSink in downstream systems, we need to understand the naming and lifecycle of the output files produced.
-
-Part files can be in one of three states:
- 1. **In-progress** : The part file that is currently being written to is in-progress
- 2. **Pending** : Once a part file is closed for writing it becomes pending
- 3. **Finished** : On successful checkpoints pending files become finished
-
-Only finished files are safe to read by downstream systems as those are guaranteed to not be modified later. Finished files can be distinguished by their naming scheme only.
-
-File naming schemes:
- - **In-progress / Pending**: `part-subtaskIndex-partFileIndex.inprogress.uid`
- - **Finished:** `part-subtaskIndex-partFileIndex`
-
-Part file indexes are strictly increasing for any given subtask (in the order they were created). However these indexes are not always sequential. When the job restarts, the next part index for all subtask will be the `max part index + 1`.
-
-Each writer subtask will have a single in-progress part file at any given time for every active bucket, but there can be several pending and finished files.
-
-**Part file example**
-
-To better understand the lifecycle of these files let's look at a simple example with 2 sink subtasks:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- └── part-1-0.inprogress.ea65a428-a1d0-4a0b-bbc5-7a436a75e575
-```
-
-When the part file `part-1-0` is rolled (let's say it becomes too large), it becomes pending but it is not renamed. The sink then opens a new part file: `part-1-1`:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- ├── part-1-0.inprogress.ea65a428-a1d0-4a0b-bbc5-7a436a75e575
- └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
-```
-
-As `part-1-0` is now pending completion, after the next successful checkpoint, it is finalized:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- ├── part-1-0
- └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
-```
-
-New buckets are created as dictated by the bucketing policy, and this doesn't affect currently in-progress files:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- ├── part-1-0
- └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
-└── 2019-08-25--13
- └── part-0-2.inprogress.2b475fec-1482-4dea-9946-eb4353b475f1
-```
-
-Old buckets can still receive new records as the bucketing policy is evaluated on a per-record basis.
## File Formats
@@ -222,10 +136,16 @@ specifying an `Encoder` we have to specify [BulkWriter.Factory]({{ site.javadocs
The `BulkWriter` logic defines how new elements added, flushed and how the bulk of records
are finalized for further encoding purposes.
-Flink comes with two built-in BulkWriter factories:
+Flink comes with three built-in BulkWriter factories:
- [ParquetWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/parquet/ParquetWriterFactory.html)
- [SequenceFileWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.html)
+ - [CompressWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/compress/CompressWriterFactory.html)
+
+
+
+ IMPORTANT: Bulk Formats can only have `OnCheckpointRollingPolicy`, which rolls (ONLY) on every checkpoint.
+
#### Parquet format
@@ -346,7 +266,174 @@ input.addSink(sink)
The SequenceFileWriterFactory supports additional constructor parameters to specify compression settings.
-### Important Considerations for S3
+## Bucket Assignment
+
+The bucketing logic defines how the data will be structured into subdirectories inside the base output directory.
+
+Both row and bulk formats (see [File Formats](#file-formats)) use the [DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) as the default assigner.
+By default the `DateTimeBucketAssigner` creates hourly buckets based on the system default timezone
+with the following format: `yyyy-MM-dd--HH`. Both the date format (*i.e.* bucket size) and timezone can be
+configured manually.
+
+We can specify a custom [BucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssigner.html) by calling `.withBucketAssigner(assigner)` on the format builders.
+
+Flink comes with two built in BucketAssigners:
+
+ - [DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) : Default time based assigner
+ - [BasePathBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/BasePathBucketAssigner.html) : Assigner that stores all part files in the base path (single global bucket)
+
+## Rolling Policy
+
+The [RollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicy.html) defines when a given in-progress part file will be closed and moved to the pending and later to finished state.
+Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure.
+The Rolling Policy in combination with the checkpointing interval (pending files become finished on the next checkpoint) control how quickly
+part files become available for downstream readers and also the size and number of these parts.
+
+Flink comes with two built-in RollingPolicies:
+
+ - [DefaultRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/DefaultRollingPolicy.html)
+ - [OnCheckpointRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/OnCheckpointRollingPolicy.html)
+
+## Part file lifecycle
+
+In order to use the output of the `StreamingFileSink` in downstream systems, we need to understand the naming and lifecycle of the output files produced.
+
+Part files can be in one of three states:
+ 1. **In-progress** : The part file that is currently being written to is in-progress
+ 2. **Pending** : Closed (due to the specified rolling policy) in-progress files that are waiting to be committed
+ 3. **Finished** : On successful checkpoints pending files transition to "Finished"
+
+Only finished files are safe to read by downstream systems as those are guaranteed to not be modified later.
+
+
+ IMPORTANT: Part file indexes are strictly increasing for any given subtask (in the order they were created). However these indexes are not always sequential. When the job restarts, the next part index for all subtask will be the `max part index + 1`
+where `max` is computed across all subtasks.
+
+
+Each writer subtask will have a single in-progress part file at any given time for every active bucket, but there can be several pending and finished files.
+
+**Part file example**
+
+To better understand the lifecycle of these files let's look at a simple example with 2 sink subtasks:
+
+```
+└── 2019-08-25--12
+ ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
+ └── part-1-0.inprogress.ea65a428-a1d0-4a0b-bbc5-7a436a75e575
+```
+
+When the part file `part-1-0` is rolled (let's say it becomes too large), it becomes pending but it is not renamed. The sink then opens a new part file: `part-1-1`:
+
+```
+└── 2019-08-25--12
+ ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
+ ├── part-1-0.inprogress.ea65a428-a1d0-4a0b-bbc5-7a436a75e575
+ └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
+```
+
+As `part-1-0` is now pending completion, after the next successful checkpoint, it is finalized:
+
+```
+└── 2019-08-25--12
+ ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
+ ├── part-1-0
+ └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
+```
+
+New buckets are created as dictated by the bucketing policy, and this doesn't affect currently in-progress files:
+
+```
+└── 2019-08-25--12
+ ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
+ ├── part-1-0
+ └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
+└── 2019-08-25--13
+ └── part-0-2.inprogress.2b475fec-1482-4dea-9946-eb4353b475f1
+```
+
+Old buckets can still receive new records as the bucketing policy is evaluated on a per-record basis.
+
+### Part file configuration
+
+Finished files can be distinguished from the in-progress ones by their naming scheme only.
+
+By default, the file naming strategy is as follows:
+ - **In-progress / Pending**: `part--.inprogress.uid`
+ - **Finished:** `part--`
+
+Flink allows the user to specify a prefix and/or a suffix for his/her part files.
+This can be done using an `OutputFileConfig`.
+For example for a prefix "prefix" and a suffix ".ext" the sink will create the following files:
+
+```
+└── 2019-08-25--12
+ ├── prefix-0-0.ext
+ ├── prefix-0-1.ext.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
+ ├── prefix-1-0.ext
+ └── prefix-1-1.ext.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
+```
+
+The user can specify an `OutputFileConfig` in the following way:
+
+
+
+## Important Considerations
+
+### General
+
+Important Note 1: When using Hadoop < 2.7, please use
+the `OnCheckpointRollingPolicy` which rolls part files on every checkpoint. The reason is that if part files "traverse"
+the checkpoint interval, then, upon recovery from a failure the `StreamingFileSink` may use the `truncate()` method of the
+filesystem to discard uncommitted data from the in-progress file. This method is not supported by pre-2.7 Hadoop versions
+and Flink will throw an exception.
+
+Important Note 2: Given that Flink sinks and UDFs in general do not differentiate between
+normal job termination (*e.g.* finite input stream) and termination due to failure, upon normal termination of a job, the last
+in-progress files will not be transitioned to the "finished" state.
+
+Important Note 3: Flink and the `StreamingFileSink` never overwrites committed data.
+Given this, when trying to restore from an old checkpoint/savepoint which assumes an in-progress file which was committed
+by subsequent successful checkpoints, Flink will refuse to resume and it will throw an exception as it cannot locate the
+in-progress file.
+
+### S3-specific
Important Note 1: For S3, the `StreamingFileSink`
supports only the [Hadoop-based](https://hadoop.apache.org/) FileSystem implementation, not
diff --git a/docs/dev/connectors/streamfile_sink.zh.md b/docs/dev/connectors/streamfile_sink.zh.md
index d78dceefef352c632133e34f6125f19f8903c4de..fdf678e34e2cd24ca97937df837f8b613411081e 100644
--- a/docs/dev/connectors/streamfile_sink.zh.md
+++ b/docs/dev/connectors/streamfile_sink.zh.md
@@ -28,9 +28,9 @@ under the License.
这个连接器提供了一个 Sink 来将分区文件写入到支持 [Flink `FileSystem`]({{ site.baseurl}}/zh/ops/filesystems/index.html) 接口的文件系统中。
-为了处理无界的流数据，Streaming File Sink 会将数据写入到桶中。如何分桶是可以配置的，默认策略是基于时间的分桶，这种策略每个小时创建并写入一个新的桶，从而得到流数据在特定时间间隔内接收记录所对应的文件。
+Streaming File Sink 会将数据写入到桶中。由于输入流可能是无界的，因此每个桶中的数据被划分为多个有限大小的文件。如何分桶是可以配置的，默认使用基于时间的分桶策略，这种策略每个小时创建一个新的桶，桶中包含的文件将记录所有该小时内从流中接收到的数据。
-桶目录中包含多个实际输出数据的部分文件（part file），对于每一个接收桶数据的 Sink Subtask ，至少存在一个部分文件（part file）。额外的部分文件（part file）将根据滚动策略创建，滚动策略是可以配置的。默认的策略是根据文件大小和超时时间来滚动文件。超时时间指打开文件的最长持续时间，以及文件关闭前的最长非活动时间。
+桶目录中的实际输出数据会被划分为多个部分文件（part file），每一个接收桶数据的 Sink Subtask ，至少包含一个部分文件（part file）。额外的部分文件（part file）将根据滚动策略创建，滚动策略是可以配置的。默认的策略是根据文件大小和超时时间来滚动文件。超时时间指打开文件的最长持续时间，以及文件关闭前的最长非活动时间。
+
+## 重要注意事项
+
+### 通用注意事项
+
+重要提示 1: 使用 Hadoop < 2.7 时，请使用 `OnCheckpointRollingPolicy` 滚动策略，该策略会在每次检查点时进行文件切割。
+这样做的原因是如果部分文件的生命周期跨多个检查点，当 `StreamingFileSink` 从之前的检查点进行恢复时会调用文件系统的 `truncate()` 方法清理 in-progress 文件中未提交的数据。
+Hadoop 2.7 之前的版本不支持这个方法，因此 Flink 会报异常。
+
+重要提示 2: 鉴于 Flink 的 sink 以及 UDF 通常不会区分作业的正常结束（比如有限流）和异常终止，因此正常结束作业的最后一批 in-progress 文件不会被转换到 "完成" 状态。
+
+重要提示 3: Flink 以及 `StreamingFileSink` 不会覆盖已经提交的数据。因此如果尝试从一个包含 in-progress 文件的旧 checkpoint/savepoint 恢复，
+且这些 in-progress 文件会被接下来的成功 checkpoint 提交，Flink 会因为无法找到 in-progress 文件而抛异常，从而恢复失败。
+
+### S3 特有的注意事项
重要提示 1: 对于 S3，`StreamingFileSink` 只支持基于 [Hadoop](https://hadoop.apache.org/)
的文件系统实现，不支持基于 [Presto](https://prestodb.io/) 的实现。如果想使用 `StreamingFileSink` 向 S3 写入数据并且将
diff --git a/docs/dev/datastream_api.md b/docs/dev/datastream_api.md
index b2c8cb81c2c973e168f572cbca390537a5a5c4be..cb19977734e740b687ae02c68bd7dcfc578d5b55 100644
--- a/docs/dev/datastream_api.md
+++ b/docs/dev/datastream_api.md
@@ -193,7 +193,7 @@ Collection-based:
Custom:
- `addSource` - Attach a new source function. For example, to read from Apache Kafka you can use
- `addSource(new FlinkKafkaConsumer08<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/index.html) for more details.
+ `addSource(new FlinkKafkaConsumer010<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/index.html) for more details.
@@ -251,7 +251,7 @@ Collection-based:
Custom:
- `addSource` - Attach a new source function. For example, to read from Apache Kafka you can use
- `addSource(new FlinkKafkaConsumer08<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/) for more details.
+ `addSource(new FlinkKafkaConsumer010<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/) for more details.
diff --git a/docs/dev/datastream_api.zh.md b/docs/dev/datastream_api.zh.md
index 8646c29a11c6c8f56d0fc23731eed6de53de27e8..ed1afd83b7a04e8e14a33f1f4f189d58c7397e2a 100644
--- a/docs/dev/datastream_api.zh.md
+++ b/docs/dev/datastream_api.zh.md
@@ -193,7 +193,7 @@ Collection-based:
Custom:
- `addSource` - Attach a new source function. For example, to read from Apache Kafka you can use
- `addSource(new FlinkKafkaConsumer08<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/index.html) for more details.
+ `addSource(new FlinkKafkaConsumer010<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/index.html) for more details.
@@ -251,7 +251,7 @@ Collection-based:
Custom:
- `addSource` - Attach a new source function. For example, to read from Apache Kafka you can use
- `addSource(new FlinkKafkaConsumer08<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/) for more details.
+ `addSource(new FlinkKafkaConsumer010<>(...))`. See [connectors]({{ site.baseurl }}/dev/connectors/) for more details.
diff --git a/docs/dev/event_time.md b/docs/dev/event_time.md
index 579074602ee05571c51f86c8ac8b5d8edac2bd59..83cfd7459a8400bbdef3b9320d9e158741c26028 100644
--- a/docs/dev/event_time.md
+++ b/docs/dev/event_time.md
@@ -24,66 +24,20 @@ specific language governing permissions and limitations
under the License.
-->
-* toc
-{:toc}
-
-# Event Time / Processing Time / Ingestion Time
-
-Flink supports different notions of *time* in streaming programs.
-
-- **Processing time:** Processing time refers to the system time of the machine that is executing the
- respective operation.
-
- When a streaming program runs on processing time, all time-based operations (like time windows) will
- use the system clock of the machines that run the respective operator. An hourly
- processing time window will include all records that arrived at a specific operator between the
- times when the system clock indicated the full hour. For example, if an application
- begins running at 9:15am, the first hourly processing time window will include events
- processed between 9:15am and 10:00am, the next window will include events processed between 10:00am and 11:00am, and so on.
-
- Processing time is the simplest notion of time and requires no coordination between streams and machines.
- It provides the best performance and the lowest latency. However, in distributed and asynchronous
- environments processing time does not provide determinism, because it is susceptible to the speed at which
- records arrive in the system (for example from the message queue), to the speed at which the
- records flow between operators inside the system, and to outages (scheduled, or otherwise).
-
-- **Event time:** Event time is the time that each individual event occurred on its producing device.
- This time is typically embedded within the records before they enter Flink, and that *event timestamp*
- can be extracted from each record. In event time, the progress of time depends on the data,
- not on any wall clocks. Event time programs must specify how to generate *Event Time Watermarks*,
- which is the mechanism that signals progress in event time. This watermarking mechanism is
- described in a later section, [below](#event-time-and-watermarks).
-
- In a perfect world, event time processing would yield completely consistent and deterministic results, regardless of when events arrive, or their ordering.
- However, unless the events are known to arrive in-order (by timestamp), event time processing incurs some latency while waiting for out-of-order events. As it is only possible to wait for a finite period of time, this places a limit on how deterministic event time applications can be.
-
- Assuming all of the data has arrived, event time operations will behave as expected, and produce correct and consistent results even when working with out-of-order or late events, or when reprocessing historic data. For example, an hourly event time window will contain all records
- that carry an event timestamp that falls into that hour, regardless of the order in which they arrive, or when they are processed. (See the section on [late events](#late-elements) for more information.)
-
-
+In this section you will learn about writing time-aware Flink programs. Please
+take a look at [Timely Stream Processing]({{site.baseurl}}{% link
+concepts/timely-stream-processing.md %}) to learn about the concepts behind
+timely stream processing.
- Note that sometimes when event time programs are processing live data in real-time, they will use some *processing time* operations in order to guarantee that they are progressing in a timely fashion.
-
-- **Ingestion time:** Ingestion time is the time that events enter Flink. At the source operator each
- record gets the source's current time as a timestamp, and time-based operations (like time windows)
- refer to that timestamp.
-
- *Ingestion time* sits conceptually in between *event time* and *processing time*. Compared to
- *processing time*, it is slightly more expensive, but gives more predictable results. Because
- *ingestion time* uses stable timestamps (assigned once at the source), different window operations
- over the records will refer to the same timestamp, whereas in *processing time* each window operator
- may assign the record to a different window (based on the local system clock and any transport delay).
-
- Compared to *event time*, *ingestion time* programs cannot handle any out-of-order events or late data,
- but the programs don't have to specify how to generate *watermarks*.
-
- Internally, *ingestion time* is treated much like *event time*, but with automatic timestamp assignment and
- automatic watermark generation.
-
-
+For information about how to use time in Flink programs refer to
+[windowing]({{site.baseurl}}{% link dev/stream/operators/windows.md %}) and
+[ProcessFunction]({{ site.baseurl }}{% link
+dev/stream/operators/process_function.md %}).
+* toc
+{:toc}
-### Setting a Time Characteristic
+## Setting a Time Characteristic
The first part of a Flink DataStream program usually sets the base *time characteristic*. That setting
defines how data stream sources behave (for example, whether they will assign timestamps), and what notion of
@@ -103,7 +57,7 @@ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
-DataStream stream = env.addSource(new FlinkKafkaConsumer09(topic, schema, props));
+DataStream stream = env.addSource(new FlinkKafkaConsumer010(topic, schema, props));
stream
.keyBy( (event) -> event.getUser() )
@@ -122,7 +76,7 @@ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime)
// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime)
// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-val stream: DataStream[MyEvent] = env.addSource(new FlinkKafkaConsumer09[MyEvent](topic, schema, props))
+val stream: DataStream[MyEvent] = env.addSource(new FlinkKafkaConsumer010[MyEvent](topic, schema, props))
stream
.keyBy( _.getUser )
@@ -144,7 +98,6 @@ env.set_stream_time_characteristic(TimeCharacteristic.ProcessingTime)
-
Note that in order to run this example in *event time*, the program needs to either use sources
that directly define event time for the data and emit watermarks themselves, or the program must
inject a *Timestamp Assigner & Watermark Generator* after the sources. Those functions describe how to access
@@ -154,77 +107,7 @@ The section below describes the general mechanism behind *timestamps* and *water
to use timestamp assignment and watermark generation in the Flink DataStream API, please refer to
[Generating Timestamps / Watermarks]({{ site.baseurl }}/dev/event_timestamps_watermarks.html).
-
-# Event Time and Watermarks
-
-*Note: Flink implements many techniques from the Dataflow Model. For a good introduction to event time and watermarks, have a look at the articles below.*
-
- - [Streaming 101](https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101) by Tyler Akidau
- - The [Dataflow Model paper](https://research.google.com/pubs/archive/43864.pdf)
-
-
-A stream processor that supports *event time* needs a way to measure the progress of event time.
-For example, a window operator that builds hourly windows needs to be notified when event time has passed beyond the
-end of an hour, so that the operator can close the window in progress.
-
-*Event time* can progress independently of *processing time* (measured by wall clocks).
-For example, in one program the current *event time* of an operator may trail slightly behind the *processing time*
-(accounting for a delay in receiving the events), while both proceed at the same speed.
-On the other hand, another streaming program might progress through weeks of event time with only a few seconds of processing,
-by fast-forwarding through some historic data already buffered in a Kafka topic (or another message queue).
-
-------
-
-The mechanism in Flink to measure progress in event time is **watermarks**.
-Watermarks flow as part of the data stream and carry a timestamp *t*. A *Watermark(t)* declares that event time has reached time
-*t* in that stream, meaning that there should be no more elements from the stream with a timestamp *t' <= t* (i.e. events with timestamps
-older or equal to the watermark).
-
-The figure below shows a stream of events with (logical) timestamps, and watermarks flowing inline. In this example the events are in order
-(with respect to their timestamps), meaning that the watermarks are simply periodic markers in the stream.
-
-
-
-Watermarks are crucial for *out-of-order* streams, as illustrated below, where the events are not ordered by their timestamps.
-In general a watermark is a declaration that by that point in the stream, all events up to a certain timestamp should have arrived.
-Once a watermark reaches an operator, the operator can advance its internal *event time clock* to the value of the watermark.
-
-
-
-Note that event time is inherited by a freshly created stream element (or elements) from either the event that produced them or
-from watermark that triggered creation of those elements.
-
-## Watermarks in Parallel Streams
-
-Watermarks are generated at, or directly after, source functions. Each parallel subtask of a source function usually
-generates its watermarks independently. These watermarks define the event time at that particular parallel source.
-
-As the watermarks flow through the streaming program, they advance the event time at the operators where they arrive. Whenever an
-operator advances its event time, it generates a new watermark downstream for its successor operators.
-
-Some operators consume multiple input streams; a union, for example, or operators following a *keyBy(...)* or *partition(...)* function.
-Such an operator's current event time is the minimum of its input streams' event times. As its input streams
-update their event times, so does the operator.
-
-The figure below shows an example of events and watermarks flowing through parallel streams, and operators tracking event time.
-
-
-
-Note that the Kafka source supports per-partition watermarking, which you can read more about [here]({{ site.baseurl }}/dev/event_timestamps_watermarks.html#timestamps-per-kafka-partition).
-
-
-## Late Elements
-
-It is possible that certain elements will violate the watermark condition, meaning that even after the *Watermark(t)* has occurred,
-more elements with timestamp *t' <= t* will occur. In fact, in many real world setups, certain elements can be arbitrarily
-delayed, making it impossible to specify a time by which all elements of a certain event timestamp will have occurred.
-Furthermore, even if the lateness can be bounded, delaying the watermarks by too much is often not desirable, because it
-causes too much delay in the evaluation of event time windows.
-
-For this reason, streaming programs may explicitly expect some *late* elements. Late elements are elements that
-arrive after the system's event time clock (as signaled by the watermarks) has already passed the time of the late element's
-timestamp. See [Allowed Lateness]({{ site.baseurl }}/dev/stream/operators/windows.html#allowed-lateness) for more information on how to work
-with late elements in event time windows.
+{% top %}
## Idling sources
diff --git a/docs/dev/event_time.zh.md b/docs/dev/event_time.zh.md
index 579074602ee05571c51f86c8ac8b5d8edac2bd59..fcd5a14ebb803f1caa427ae42b9f97597c55972c 100644
--- a/docs/dev/event_time.zh.md
+++ b/docs/dev/event_time.zh.md
@@ -103,7 +103,7 @@ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
-DataStream stream = env.addSource(new FlinkKafkaConsumer09(topic, schema, props));
+DataStream stream = env.addSource(new FlinkKafkaConsumer010(topic, schema, props));
stream
.keyBy( (event) -> event.getUser() )
@@ -122,7 +122,7 @@ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime)
// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime)
// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-val stream: DataStream[MyEvent] = env.addSource(new FlinkKafkaConsumer09[MyEvent](topic, schema, props))
+val stream: DataStream[MyEvent] = env.addSource(new FlinkKafkaConsumer010[MyEvent](topic, schema, props))
stream
.keyBy( _.getUser )
diff --git a/docs/dev/event_timestamps_watermarks.md b/docs/dev/event_timestamps_watermarks.md
index fdb716bc8caa487ad92630264270b57b696dad58..28e1f2f343d1e83b6e774514fc1adbdfbb3e76a0 100644
--- a/docs/dev/event_timestamps_watermarks.md
+++ b/docs/dev/event_timestamps_watermarks.md
@@ -351,7 +351,7 @@ streaming dataflow in that case.
{% highlight scala %}
-val kafkaSource = new FlinkKafkaConsumer09[MyType]("myTopic", schema, props)
+val kafkaSource = new FlinkKafkaConsumer010[MyType]("myTopic", schema, props)
kafkaSource.assignTimestampsAndWatermarks(new AscendingTimestampExtractor[MyType] {
def extractAscendingTimestamp(element: MyType): Long = element.eventTimestamp
})
diff --git a/docs/dev/execution_plans.md b/docs/dev/execution_plans.md
index 29a3641c3569c8e6663de524a24ad5229c75ad64..a48adc49f9ef7dd060fa82ca40bb53c5a67bfd14 100644
--- a/docs/dev/execution_plans.md
+++ b/docs/dev/execution_plans.md
@@ -28,10 +28,8 @@ useful to know how exactly Flink will execute your program.
__Plan Visualization Tool__
-Flink comes packaged with a visualization tool for execution plans. The HTML document containing
-the visualizer is located under ```tools/planVisualizer.html```. It takes a JSON representation of
-the job execution plan and visualizes it as a graph with complete annotations of execution
-strategies.
+Flink provides a [visualization tool](https://flink.apache.org/visualizer/) for execution plans, which takes a JSON
+representation of the job execution plan and visualizes it as a graph with complete annotations of execution strategies.
The following code shows how to print the execution plan JSON from your program:
@@ -59,7 +57,7 @@ println(env.getExecutionPlan())
To visualize the execution plan, do the following:
-1. **Open** ```planVisualizer.html``` with your web browser,
+1. **Open** the [visualizer](https://flink.apache.org/visualizer/) with your web browser,
2. **Paste** the JSON string into the text field, and
3. **Press** the draw button.
@@ -71,8 +69,7 @@ After these steps, a detailed execution plan will be visualized.
__Web Interface__
Flink offers a web interface for submitting and executing jobs. The interface is part of the JobManager's
-web interface for monitoring, per default running on port 8081. Job submission via this interfaces requires
-that you have set `web.submit.enable: true` in `flink-conf.yaml`.
+web interface for monitoring, per default running on port 8081.
You may specify program arguments before the job is executed. The plan visualization enables you to show
the execution plan before executing the Flink job.
diff --git a/docs/dev/execution_plans.zh.md b/docs/dev/execution_plans.zh.md
index eefa8f4e7491334a6b72601302dd4cb335a95c87..a48adc49f9ef7dd060fa82ca40bb53c5a67bfd14 100644
--- a/docs/dev/execution_plans.zh.md
+++ b/docs/dev/execution_plans.zh.md
@@ -1,5 +1,5 @@
---
-title: "执行计划"
+title: "Execution Plans"
nav-parent_id: execution
nav-pos: 40
---
@@ -28,10 +28,8 @@ useful to know how exactly Flink will execute your program.
__Plan Visualization Tool__
-Flink comes packaged with a visualization tool for execution plans. The HTML document containing
-the visualizer is located under ```tools/planVisualizer.html```. It takes a JSON representation of
-the job execution plan and visualizes it as a graph with complete annotations of execution
-strategies.
+Flink provides a [visualization tool](https://flink.apache.org/visualizer/) for execution plans, which takes a JSON
+representation of the job execution plan and visualizes it as a graph with complete annotations of execution strategies.
The following code shows how to print the execution plan JSON from your program:
@@ -59,7 +57,7 @@ println(env.getExecutionPlan())
To visualize the execution plan, do the following:
-1. **Open** ```planVisualizer.html``` with your web browser,
+1. **Open** the [visualizer](https://flink.apache.org/visualizer/) with your web browser,
2. **Paste** the JSON string into the text field, and
3. **Press** the draw button.
@@ -71,8 +69,7 @@ After these steps, a detailed execution plan will be visualized.
__Web Interface__
Flink offers a web interface for submitting and executing jobs. The interface is part of the JobManager's
-web interface for monitoring, per default running on port 8081. Job submission via this interfaces requires
-that you have set `web.submit.enable: true` in `flink-conf.yaml`.
+web interface for monitoring, per default running on port 8081.
You may specify program arguments before the job is executed. The plan visualization enables you to show
the execution plan before executing the Flink job.
diff --git a/docs/dev/index.md b/docs/dev/index.md
index b58120471b371c97d5312cfca04c2eb557abcd32..0a5616b8092cbbeb79cb197a1313505cac9097ad 100644
--- a/docs/dev/index.md
+++ b/docs/dev/index.md
@@ -5,6 +5,7 @@ nav-title: ' Applica
nav-parent_id: root
nav-pos: 5
section-break: true
+always-expand: true
---
-Apache Flink's State Processor API provides powerful functionality to reading, writing, and modifing savepoints and checkpoints using Flink’s batch DataSet api.
-This is useful for tasks such as analyzing state for interesting patterns, troubleshooting or auditing jobs by checking for discrepancies, and bootstrapping state for new applications.
+Apache Flink's State Processor API provides powerful functionality to reading, writing, and modifing savepoints and checkpoints using Flink’s batch DataSet API.
+Due to the [interoperability of DataSet and Table API](https://ci.apache.org/projects/flink/flink-docs-master/dev/table/common.html#integration-with-datastream-and-dataset-api), you can even use relational Table API or SQL queries to analyze and process state data.
+
+For example, you can take a savepoint of a running stream processing application and analyze it with a DataSet batch program to verify that the application behaves correctly.
+Or you can read a batch of data from any store, preprocess it, and write the result to a savepoint that you use to bootstrap the state of a streaming application.
+It is also possible to fix inconsistent state entries.
+Finally, the State Processor API opens up many ways to evolve a stateful application that was previously blocked by parameter and design choices that could not be changed without losing all the state of the application after it was started.
+For example, you can now arbitrarily modify the data types of states, adjust the maximum parallelism of operators, split or merge operator state, re-assign operator UIDs, and so on.
+
+To get started with the state processor api, include the following library in your application.
+
+{% highlight xml %}
+
+ org.apache.flink
+ flink-state-processor-api{{ site.scala_version_suffix }}
+ {{site.version}}
+ provided
+
+{% endhighlight %}
* This will be replaced by the TOC
{:toc}
-## Abstraction
+## Mapping Application State to DataSets
-To understand how to best interact with savepoints in a batch context it is important to have a clear mental model of how the data in Flink state relates to a traditional relational database.
+The State Processor API maps the state of a streaming application to one or more data sets that can be processed separately.
+In order to be able to use the API, you need to understand how this mapping works.
-A database can be thought of as one or more namespaces, each containing a collection of tables.
-Those tables in turn contain columns whose values have some intrinsic relationship between them, such as being scoped under the same key.
+But let us first have a look at what a stateful Flink job looks like.
+A Flink job is composed of operators; typically one or more source operators, a few operators for the actual processing, and one or more sink operators.
+Each operator runs in parallel in one or more tasks and can work with different types of state.
+An operator can have zero, one, or more *“operator states”* which are organized as lists that are scoped to the operator's tasks.
+If the operator is applied on a keyed stream, it can also have zero, one, or more *“keyed states”* which are scoped to a key that is extracted from each processed record.
+You can think of keyed state as a distributed key-value map.
-A savepoint represents the state of a Flink job at a particular point in time which is made up of many operators.
-Those operators contain various kinds of state, both partitioned or keyed state, and non-partitioned or operator state.
+The following figure shows the application “MyApp” which consists of three operators called “Src”, “Proc”, and “Snk”.
+Src has one operator state (os1), Proc has one operator state (os2) and two keyed states (ks1, ks2) and Snk is stateless.
-
-This job contains multiple operators along with various kinds of state.
-When analyzing that state we can first scope data by its operator, named by setting its uid.
-Within each operator we can look at the registered states.
-`CurrencyConverter` has a broadcast state, which is a type of non-partitioned operator state.
-In general, there is no relationship between any two elements in an operator state and so we can look at each value as being its own row.
-Contrast this with Summarize, which contains two keyed states.
-Because both states are scoped under the same key we can safely assume there exists some relationship between the two values.
-Therefore, keyed state is best understood as a single table per operator containing one _key_ column along with _n_ value columns, one for each registered state.
-All of this means that the state for this job could be described using the following pseudo-sql commands.
+A savepoint or checkpoint of MyApp consists of the data of all states, organized in a way that the states of each task can be restored.
+When processing the data of a savepoint (or checkpoint) with a batch job, we need a mental model that maps the data of the individual tasks' states into data sets or tables.
+In fact, we can think of a savepoint as a database. Every operator (identified by its UID) represents a namespace.
+Each operator state of an operator is mapped to a dedicated table in the namespace with a single column that holds the state's data of all tasks.
+All keyed states of an operator are mapped to a single table consisting of a column for the key, and one column for each keyed state.
+The following figure shows how a savepoint of MyApp is mapped to a database.
-{% highlight sql %}
-CREATE NAMESPACE currency_converter;
-
-CREATE TABLE currency_converter.rates (
- value Tuple2
-);
-
-CREATE NAMESPACE summarize;
-
-CREATE TABLE summarize.keyed_state (
- key INTEGER PRIMARY KEY,
- total DOUBLE,
- count INTEGER
-);
-{% endhighlight %}
-
-In general, the savepoint ↔ database relationship can be summarized as:
+
+
+
- * A savepoint is a database
- * An operator is a namespace named by its uid
- * Each operator state represents a single table
- * Each element in an operator state represents a single row in that table
- * Each operator containing keyed state has a single “keyed_state” table
- * Each keyed_state table has one key column mapping the key value of the operator
- * Each registered state represents a single column in the table
- * Each row in the table maps to a single key
+The figure shows how the values of Src's operator state are mapped to a table with one column and five rows, one row for each of the list entries across all parallel tasks of Src.
+Operator state os2 of the operator “Proc” is similarly mapped to an individual table.
+The keyed states ks1 and ks2 are combined to a single table with three columns, one for the key, one for ks1 and one for ks2.
+The keyed table holds one row for each distinct key of both keyed states.
+Since the operator “Snk” does not have any state, its namespace is empty.
## Reading State
-Reading state begins by specifiying the path to a valid savepoint or checkpoint along with the `StateBackend` that should be used to restore the data.
-The compatability guarantees for restoring state are identical to those when restoring a `DataStream` application.
+Reading state begins by specifying the path to a valid savepoint or checkpoint along with the `StateBackend` that should be used to restore the data.
+The compatibility guarantees for restoring state are identical to those when restoring a `DataStream` application.
-When reading operator state, simply specify the operator uid, state name, and type information.
+### Operator State
+
+[Operator state]({{ site.baseurl }}/dev/stream/state/state.html#operator-state) is any non-keyed state in Flink.
+This includes, but is not limited to, any use of `CheckpointedFunction` or `BroadcastState` within an application.
+When reading operator state, users specify the operator uid, the state name, and the type information.
+
+#### Operator List State
+
+Operator state stored in a `CheckpointedFunction` using `getListState` can be read using `ExistingSavepoint#readListState`.
+The state name and type information should match those used to define the `ListStateDescriptor` that declared this state in the DataStream application.
-val unionState = savepoint.readUnionState(
+#### Operator Union List State
+
+Operator state stored in a `CheckpointedFunction` using `getUnionListState` can be read using `ExistingSavepoint#readUnionState`.
+The state name and type information should match those used to define the `ListStateDescriptor` that declared this state in the DataStream application.
+The framework will return a _single_ copy of the state, equivalent to restoring a DataStream with parallelism 1.
+
+
+
+
+#### Broadcast State
+
+[BroadcastState]({{ site.baseurl }} /dev/stream/state/broadcast_state.html) can be read using `ExistingSavepoint#readBroadcastState`.
+The state name and type information should match those used to define the `MapStateDescriptor` that declared this state in the DataStream application.
+The framework will return a _single_ copy of the state, equivalent to restoring a DataStream with parallelism 1.
+
+
+{% highlight scala %}
val broadcastState = savepoint.readBroadcastState(
"my-uid",
"broadcast-state",
@@ -206,12 +185,14 @@ val broadcastState = savepoint.readBroadcastState(
-A custom `TypeSerializer` may also be specified if one was used in the `StateDescriptor` for the state.
+#### Using Custom Serializers
+
+Each of the operator state readers support using custom `TypeSerializers` if one was used to define the `StateDescriptor` that wrote out the state.
-When reading keyed state, users specify a KeyedStateReaderFunction to allow reading arbitrary columns and complex state types such as ListState, MapState, and AggregatingState.
+### Keyed State
+
+[Keyed state]({{ site.baseurl }}/dev/stream/state/state.html#keyed-state), or partitioned state, is any state that is partitioned relative to a key.
+When reading a keyed state, users specify the operator id and a `KeyedStateReaderFunction`.
+
+The `KeyedStateReaderFunction` allows users to read arbitrary columns and complex state types such as ListState, MapState, and AggregatingState.
This means if an operator contains a stateful process function such as:
@@ -239,55 +225,80 @@ public class StatefulFunctionWithTime extends KeyedProcessFunction state;
+ ListState updateTimes;
+
@Override
public void open(Configuration parameters) {
ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT);
state = getRuntimeContext().getState(stateDescriptor);
+
+ ListStateDescriptor updateDescriptor = new ListStateDescriptor<>("times", Types.LONG);
+ updateTimes = getRuntimeContext().getListState(updateDescriptor);
}
@Override
public void processElement(Integer value, Context ctx, Collector out) throws Exception {
state.update(value + 1);
+ updateTimes.add(System.currentTimeMillis());
}
}
{% endhighlight %}
{% highlight scala %}
-public class StatefulFunctionWithTime extends KeyedProcessFunction[Integer, Integer, Void] {
+class StatefulFunctionWithTime extends KeyedProcessFunction[Integer, Integer, Void] {
- var state: ValueState[Integer];
+ var state: ValueState[Integer] = _
- override def open(parameters: Configuration) {
- val stateDescriptor = new ValueStateDescriptor("state", Types.INT);
- state = getRuntimeContext().getState(stateDescriptor);
+ var updateTimes: ListState[Long] = _
+
+ @throws[Exception]
+ override def open(parameters: Configuration): Unit = {
+ val stateDescriptor = new ValueStateDescriptor("state", Types.INT)
+ state = getRuntimeContext().getState(stateDescriptor)
+
+ val updateDescriptor = new ListStateDescriptor("times", Types.LONG)
+ updateTimes = getRuntimeContext().getListState(updateDescriptor)
}
- override def processElement(value: Integer, ctx: Context, out: Collector[Void]) {
- state.update(value + 1);
+ @throws[Exception]
+ override def processElement(value: Integer, ctx: KeyedProcessFunction[ Integer, Integer, Void ]#Context, out: Collector[Void]): Unit = {
+ state.update(value + 1)
+ updateTimes.add(System.currentTimeMillis)
}
}
{% endhighlight %}
-Then it can read by defining an output type and corresponding KeyedStateReaderFunction.
+Then it can read by defining an output type and corresponding `KeyedStateReaderFunction`.
{% highlight java %}
-class KeyedState {
- Integer key;
- Integer value;
+DataSet keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction());
+
+public class KeyedState {
+ public int key;
+
+ public int value;
+
+ public List times;
}
-class ReaderFunction extends KeyedStateReaderFunction {
+public class ReaderFunction extends KeyedStateReaderFunction {
+
ValueState state;
+ ListState updateTimes;
+
@Override
public void open(Configuration parameters) {
- ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT);
- state = getRuntimeContext().getState(stateDescriptor);
+ ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT);
+ state = getRuntimeContext().getState(stateDescriptor);
+
+ ListStateDescriptor updateDescriptor = new ListStateDescriptor<>("times", Types.LONG);
+ updateTimes = getRuntimeContext().getListState(updateDescriptor);
}
@Override
@@ -295,58 +306,221 @@ class ReaderFunction extends KeyedStateReaderFunction {
Integer key,
Context ctx,
Collector out) throws Exception {
-
- KeyedState data = new KeyedState();
- data.key = key;
- data.value = state.value();
- out.collect(data);
+
+ KeyedState data = new KeyedState();
+ data.key = key;
+ data.value = state.value();
+ data.times = StreamSupport
+ .stream(updateTimes.get().spliterator(), false)
+ .collect(Collectors.toList());
+
+ out.collect(data);
}
}
-
-DataSet keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction());
{% endhighlight %}
{% highlight scala %}
-case class KeyedState(key: Int, value: Int)
+val keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction)
+
+case class KeyedState(key: Int, value: Int, times: List[Long])
class ReaderFunction extends KeyedStateReaderFunction[Integer, KeyedState] {
- var state: ValueState[Integer];
- override def open(parameters: Configuration) {
- val stateDescriptor = new ValueStateDescriptor("state", Types.INT);
- state = getRuntimeContext().getState(stateDescriptor);
- }
+ var state: ValueState[Integer] = _
+
+ var updateTimes: ListState[Long] = _
+ @throws[Exception]
+ override def open(parameters: Configuration): Unit = {
+ val stateDescriptor = new ValueStateDescriptor("state", Types.INT)
+ state = getRuntimeContext().getState(stateDescriptor)
+
+ val updateDescriptor = new ListStateDescriptor("times", Types.LONG)
+ updateTimes = getRuntimeContext().getListState(updateDescriptor)
+ }
+
+
+ @throws[Exception]
override def processKey(
key: Int,
ctx: Context,
- out: Collector[Keyedstate]) throws Exception {
+ out: Collector[Keyedstate]): Unit = {
- val data = KeyedState(key, state.value())
- out.collect(data);
+ val data = KeyedState(key, state.value(), updateTimes.get.asScala.toList)
+ out.collect(data)
}
}
-
-val keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction());
{% endhighlight %}
-{% panel **Note:** When using a `KeyedStateReaderFunction` all state descriptors must be registered eagerly inside of open. Any attempt to call `RuntimeContext#getState`, `RuntimeContext#getListState`, or `RuntimeContext#getMapState` will result in a `RuntimeException`. %}
+Along with reading registered state values, each key has access to a `Context` with metadata such as registered event time and processing time timers.
+
+{% panel **Note:** When using a `KeyedStateReaderFunction`, all state descriptors must be registered eagerly inside of open. Any attempt to call a `RuntimeContext#get*State` will result in a `RuntimeException`. %}
## Writing New Savepoints
-State writers are based around the abstraction of `Savepoint`, where one `Savepoint` may have many operators and the state for any particular operator is created using a `BootstrapTransformation`.
+`Savepoint`'s may also be written, which allows such use cases as bootstrapping state based on historical data.
+Each savepoint is made up of one or more `BootstrapTransformation`'s (explained below), each of which defines the state for an individual operator.
-A `BootstrapTransformation` starts with a `DataSet` containing the values that are to be written into state.
-The transformation may be optionally `keyed` depending on whether or not you are writing keyed or operator state.
-Finally a bootstrap function is applied depending to the transformation; Flink supplies `KeyedStateBootstrapFunction` for writing keyed state, `StateBootstrapFunction` for writing non keyed state, and `BroadcastStateBootstrapFunction` for writing broadcast state.
+
+
+The [UIDs]({{ site.baseurl}}/ops/state/savepoints.html#assigning-operator-ids) associated with each operator must match one to one with the UIDs assigned to the operators in your `DataStream` application; these are how Flink knows what state maps to which operator.
+
+### Operator State
+
+Simple operator state, using `CheckpointedFunction`, can be created using the `StateBootstrapFunction`.
+
+
+
+### Broadcast State
+
+[BroadcastState]({{ site.baseurl }} /dev/stream/state/broadcast_state.html) can be written using a `BroadcastStateBootstrapFunction`. Similar to broadcast state in the `DataStream` API, the full state must fit in memory.
{% highlight java %}
-public class Account {
+public class CurrencyRate {
+ public String currency;
+
+ public Double rate;
+}
+
+public class CurrencyBootstrapFunction extends BroadcastStateBootstrapFunction {
+
+ public static final MapStateDescriptor descriptor =
+ new MapStateDescriptor<>("currency-rates", Types.STRING, Types.DOUBLE);
+
+ @Override
+ public void processElement(CurrencyRate value, Context ctx) throws Exception {
+ ctx.getBroadcastState(descriptor).put(value.currency, value.rate);
+ }
+}
+
+DataSet currencyDataSet = bEnv.fromCollection(
+ new CurrencyRate("USD", 1.0), new CurrencyRate("EUR", 1.3));
+
+BootstrapTransformation broadcastTransformation = OperatorTransformation
+ .bootstrapWith(currencyDataSet)
+ .transform(new CurrencyBootstrapFunction());
+{% endhighlight %}
+
+
+### Keyed State
+
+Keyed state for `ProcessFunction`'s and other `RichFunction` types can be written using a `KeyedStateBootstrapFunction`.
+
+
+
+{% highlight java %}
+public class Account {
public int id;
public double amount;
@@ -382,12 +556,13 @@ BootstrapTransformation transformation = OperatorTransformation
{% highlight scala %}
case class Account(id: Int, amount: Double, timestamp: Long)
-
+
class AccountBootstrapper extends KeyedStateBootstrapFunction[Integer, Account] {
var state: ValueState[Double]
+ @throws[Exception]
override def open(parameters: Configuration): Unit = {
- val descriptor = new ValueStateDescriptor[Double]("total",Types.DOUBLE)
+ val descriptor = new ValueStateDescriptor("total",Types.DOUBLE)
state = getRuntimeContext().getState(descriptor)
}
@@ -403,47 +578,27 @@ val accountDataSet = bEnv.fromCollection(accounts)
val transformation = OperatorTransformation
.bootstrapWith(accountDataSet)
- .keyBy(acc -> acc.id)
- .transform(new AccountBootstrapper())
+ .keyBy(acc => acc.id)
+ .transform(new AccountBootstrapper)
{% endhighlight %}
The `KeyedStateBootstrapFunction` supports setting event time and processing time timers.
The timers will not fire inside the bootstrap function and only become active once restored within a `DataStream` application.
-If a processing time timer is set but the state is not restored until after that time has passed, the timer will fire immediatly upon start.
+If a processing time timer is set but the state is not restored until after that time has passed, the timer will fire immediately upon start.
-Once one or more transformations have been created they may be combined into a single `Savepoint`.
-`Savepoint`'s are created using a state backend and max parallelism, they may contain any number of operators.
+Attention If your bootstrap function creates timers, the state can only be restored using one of the [process]({{ site.baseurl }}/dev/stream/operators/process_function.html) type functions.
-
-
-Besides creating a savepoint from scratch, you can base on off an existing savepoint such as when bootstrapping a single new operator for an existing job.
+## Modifying Savepoints
+
+Besides creating a savepoint from scratch, you can base one off an existing savepoint such as when bootstrapping a single new operator for an existing job.
{% highlight scala %}
Savepoint
- .load(backend, oldPath)
+ .load(bEnv, new MemoryStateBackend, oldPath)
.withOperator("uid", transformation)
.write(newPath)
{% endhighlight %}
diff --git a/docs/dev/libs/state_processor_api.zh.md b/docs/dev/libs/state_processor_api.zh.md
index acde2957205a4e0e3471a577b430f3dfc3439085..bbaadde725f2862ffa6e5763f4ad55f5cecf22d4 100644
--- a/docs/dev/libs/state_processor_api.zh.md
+++ b/docs/dev/libs/state_processor_api.zh.md
@@ -23,166 +23,103 @@ specific language governing permissions and limitations
under the License.
-->
-Apache Flink's State Processor API provides powerful functionality to reading, writing, and modifing savepoints and checkpoints using Flink’s batch DataSet api.
-This is useful for tasks such as analyzing state for interesting patterns, troubleshooting or auditing jobs by checking for discrepancies, and bootstrapping state for new applications.
+Apache Flink's State Processor API provides powerful functionality to reading, writing, and modifing savepoints and checkpoints using Flink’s batch DataSet API.
+Due to the [interoperability of DataSet and Table API](https://ci.apache.org/projects/flink/flink-docs-master/dev/table/common.html#integration-with-datastream-and-dataset-api), you can even use relational Table API or SQL queries to analyze and process state data.
+
+For example, you can take a savepoint of a running stream processing application and analyze it with a DataSet batch program to verify that the application behaves correctly.
+Or you can read a batch of data from any store, preprocess it, and write the result to a savepoint that you use to bootstrap the state of a streaming application.
+It is also possible to fix inconsistent state entries.
+Finally, the State Processor API opens up many ways to evolve a stateful application that was previously blocked by parameter and design choices that could not be changed without losing all the state of the application after it was started.
+For example, you can now arbitrarily modify the data types of states, adjust the maximum parallelism of operators, split or merge operator state, re-assign operator UIDs, and so on.
+
+To get started with the state processor api, include the following library in your application.
+
+{% highlight xml %}
+
+ org.apache.flink
+ flink-state-processor-api{{ site.scala_version_suffix }}
+ {{site.version}}
+ provided
+
+{% endhighlight %}
* This will be replaced by the TOC
{:toc}
-## Abstraction
+## Mapping Application State to DataSets
-To understand how to best interact with savepoints in a batch context it is important to have a clear mental model of how the data in Flink state relates to a traditional relational database.
+The State Processor API maps the state of a streaming application to one or more data sets that can be processed separately.
+In order to be able to use the API, you need to understand how this mapping works.
-A database can be thought of as one or more namespaces, each containing a collection of tables.
-Those tables in turn contain columns whose values have some intrinsic relationship between them, such as being scoped under the same key.
+But let us first have a look at what a stateful Flink job looks like.
+A Flink job is composed of operators; typically one or more source operators, a few operators for the actual processing, and one or more sink operators.
+Each operator runs in parallel in one or more tasks and can work with different types of state.
+An operator can have zero, one, or more *“operator states”* which are organized as lists that are scoped to the operator's tasks.
+If the operator is applied on a keyed stream, it can also have zero, one, or more *“keyed states”* which are scoped to a key that is extracted from each processed record.
+You can think of keyed state as a distributed key-value map.
-A savepoint represents the state of a Flink job at a particular point in time which is made up of many operators.
-Those operators contain various kinds of state, both partitioned or keyed state, and non-partitioned or operator state.
+The following figure shows the application “MyApp” which consists of three operators called “Src”, “Proc”, and “Snk”.
+Src has one operator state (os1), Proc has one operator state (os2) and two keyed states (ks1, ks2) and Snk is stateless.
-
-This job contains multiple operators along with various kinds of state.
-When analyzing that state we can first scope data by its operator, named by setting its uid.
-Within each operator we can look at the registered states.
-`CurrencyConverter` has a broadcast state, which is a type of non-partitioned operator state.
-In general, there is no relationship between any two elements in an operator state and so we can look at each value as being its own row.
-Contrast this with Summarize, which contains two keyed states.
-Because both states are scoped under the same key we can safely assume there exists some relationship between the two values.
-Therefore, keyed state is best understood as a single table per operator containing one _key_ column along with _n_ value columns, one for each registered state.
-All of this means that the state for this job could be described using the following pseudo-sql commands.
+A savepoint or checkpoint of MyApp consists of the data of all states, organized in a way that the states of each task can be restored.
+When processing the data of a savepoint (or checkpoint) with a batch job, we need a mental model that maps the data of the individual tasks' states into data sets or tables.
+In fact, we can think of a savepoint as a database. Every operator (identified by its UID) represents a namespace.
+Each operator state of an operator is mapped to a dedicated table in the namespace with a single column that holds the state's data of all tasks.
+All keyed states of an operator are mapped to a single table consisting of a column for the key, and one column for each keyed state.
+The following figure shows how a savepoint of MyApp is mapped to a database.
-{% highlight sql %}
-CREATE NAMESPACE currency_converter;
-
-CREATE TABLE currency_converter.rates (
- value Tuple2
-);
-
-CREATE NAMESPACE summarize;
-
-CREATE TABLE summarize.keyed_state (
- key INTEGER PRIMARY KEY,
- total DOUBLE,
- count INTEGER
-);
-{% endhighlight %}
-
-In general, the savepoint ↔ database relationship can be summarized as:
+
+
+
- * A savepoint is a database
- * An operator is a namespace named by its uid
- * Each operator state represents a single table
- * Each element in an operator state represents a single row in that table
- * Each operator containing keyed state has a single “keyed_state” table
- * Each keyed_state table has one key column mapping the key value of the operator
- * Each registered state represents a single column in the table
- * Each row in the table maps to a single key
+The figure shows how the values of Src's operator state are mapped to a table with one column and five rows, one row for each of the list entries across all parallel tasks of Src.
+Operator state os2 of the operator “Proc” is similarly mapped to an individual table.
+The keyed states ks1 and ks2 are combined to a single table with three columns, one for the key, one for ks1 and one for ks2.
+The keyed table holds one row for each distinct key of both keyed states.
+Since the operator “Snk” does not have any state, its namespace is empty.
## Reading State
-Reading state begins by specifiying the path to a valid savepoint or checkpoint along with the `StateBackend` that should be used to restore the data.
-The compatability guarantees for restoring state are identical to those when restoring a `DataStream` application.
+Reading state begins by specifying the path to a valid savepoint or checkpoint along with the `StateBackend` that should be used to restore the data.
+The compatibility guarantees for restoring state are identical to those when restoring a `DataStream` application.
-When reading operator state, simply specify the operator uid, state name, and type information.
+### Operator State
+
+[Operator state]({{ site.baseurl }}/dev/stream/state/state.html#operator-state) is any non-keyed state in Flink.
+This includes, but is not limited to, any use of `CheckpointedFunction` or `BroadcastState` within an application.
+When reading operator state, users specify the operator uid, the state name, and the type information.
+
+#### Operator List State
+
+Operator state stored in a `CheckpointedFunction` using `getListState` can be read using `ExistingSavepoint#readListState`.
+The state name and type information should match those used to define the `ListStateDescriptor` that declared this state in the DataStream application.
-val unionState = savepoint.readUnionState(
+#### Operator Union List State
+
+Operator state stored in a `CheckpointedFunction` using `getUnionListState` can be read using `ExistingSavepoint#readUnionState`.
+The state name and type information should match those used to define the `ListStateDescriptor` that declared this state in the DataStream application.
+The framework will return a _single_ copy of the state, equivalent to restoring a DataStream with parallelism 1.
+
+
+
+
+#### Broadcast State
+
+[BroadcastState]({{ site.baseurl }} /dev/stream/state/broadcast_state.html) can be read using `ExistingSavepoint#readBroadcastState`.
+The state name and type information should match those used to define the `MapStateDescriptor` that declared this state in the DataStream application.
+The framework will return a _single_ copy of the state, equivalent to restoring a DataStream with parallelism 1.
+
+
+{% highlight scala %}
val broadcastState = savepoint.readBroadcastState(
"my-uid",
"broadcast-state",
@@ -206,12 +185,14 @@ val broadcastState = savepoint.readBroadcastState(
-A custom `TypeSerializer` may also be specified if one was used in the `StateDescriptor` for the state.
+#### Using Custom Serializers
+
+Each of the operator state readers support using custom `TypeSerializers` if one was used to define the `StateDescriptor` that wrote out the state.
-When reading keyed state, users specify a KeyedStateReaderFunction to allow reading arbitrary columns and complex state types such as ListState, MapState, and AggregatingState.
+### Keyed State
+
+[Keyed state]({{ site.baseurl }}/dev/stream/state/state.html#keyed-state), or partitioned state, is any state that is partitioned relative to a key.
+When reading a keyed state, users specify the operator id and a `KeyedStateReaderFunction`.
+
+The `KeyedStateReaderFunction` allows users to read arbitrary columns and complex state types such as ListState, MapState, and AggregatingState.
This means if an operator contains a stateful process function such as:
@@ -239,55 +225,80 @@ public class StatefulFunctionWithTime extends KeyedProcessFunction state;
+ ListState updateTimes;
+
@Override
public void open(Configuration parameters) {
ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT);
state = getRuntimeContext().getState(stateDescriptor);
+
+ ListStateDescriptor updateDescriptor = new ListStateDescriptor<>("times", Types.LONG);
+ updateTimes = getRuntimeContext().getListState(updateDescriptor);
}
@Override
public void processElement(Integer value, Context ctx, Collector out) throws Exception {
state.update(value + 1);
+ updateTimes.add(System.currentTimeMillis());
}
}
{% endhighlight %}
{% highlight scala %}
-public class StatefulFunctionWithTime extends KeyedProcessFunction[Integer, Integer, Void] {
+class StatefulFunctionWithTime extends KeyedProcessFunction[Integer, Integer, Void] {
- var state: ValueState[Integer];
+ var state: ValueState[Integer] = _
- override def open(parameters: Configuration) {
- val stateDescriptor = new ValueStateDescriptor("state", Types.INT);
- state = getRuntimeContext().getState(stateDescriptor);
+ var updateTimes: ListState[Long] = _
+
+ @throws[Exception]
+ override def open(parameters: Configuration): Unit = {
+ val stateDescriptor = new ValueStateDescriptor("state", Types.INT)
+ state = getRuntimeContext().getState(stateDescriptor)
+
+ val updateDescriptor = new ListStateDescriptor("times", Types.LONG)
+ updateTimes = getRuntimeContext().getListState(updateDescriptor)
}
- override def processElement(value: Integer, ctx: Context, out: Collector[Void]) {
- state.update(value + 1);
+ @throws[Exception]
+ override def processElement(value: Integer, ctx: KeyedProcessFunction[ Integer, Integer, Void ]#Context, out: Collector[Void]): Unit = {
+ state.update(value + 1)
+ updateTimes.add(System.currentTimeMillis)
}
}
{% endhighlight %}
-Then it can read by defining an output type and corresponding KeyedStateReaderFunction.
+Then it can read by defining an output type and corresponding `KeyedStateReaderFunction`.
{% highlight java %}
-class KeyedState {
- Integer key;
- Integer value;
+DataSet keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction());
+
+public class KeyedState {
+ public int key;
+
+ public int value;
+
+ public List times;
}
-class ReaderFunction extends KeyedStateReaderFunction {
+public class ReaderFunction extends KeyedStateReaderFunction {
+
ValueState state;
+ ListState updateTimes;
+
@Override
public void open(Configuration parameters) {
- ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT);
- state = getRuntimeContext().getState(stateDescriptor);
+ ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT);
+ state = getRuntimeContext().getState(stateDescriptor);
+
+ ListStateDescriptor updateDescriptor = new ListStateDescriptor<>("times", Types.LONG);
+ updateTimes = getRuntimeContext().getListState(updateDescriptor);
}
@Override
@@ -295,58 +306,221 @@ class ReaderFunction extends KeyedStateReaderFunction {
Integer key,
Context ctx,
Collector out) throws Exception {
-
- KeyedState data = new KeyedState();
- data.key = key;
- data.value = state.value();
- out.collect(data);
+
+ KeyedState data = new KeyedState();
+ data.key = key;
+ data.value = state.value();
+ data.times = StreamSupport
+ .stream(updateTimes.get().spliterator(), false)
+ .collect(Collectors.toList());
+
+ out.collect(data);
}
}
-
-DataSet keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction());
{% endhighlight %}
{% highlight scala %}
-case class KeyedState(key: Int, value: Int)
+val keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction)
+
+case class KeyedState(key: Int, value: Int, times: List[Long])
class ReaderFunction extends KeyedStateReaderFunction[Integer, KeyedState] {
- var state: ValueState[Integer];
- override def open(parameters: Configuration) {
- val stateDescriptor = new ValueStateDescriptor("state", Types.INT);
- state = getRuntimeContext().getState(stateDescriptor);
- }
+ var state: ValueState[Integer] = _
+
+ var updateTimes: ListState[Long] = _
+ @throws[Exception]
+ override def open(parameters: Configuration): Unit = {
+ val stateDescriptor = new ValueStateDescriptor("state", Types.INT)
+ state = getRuntimeContext().getState(stateDescriptor)
+
+ val updateDescriptor = new ListStateDescriptor("times", Types.LONG)
+ updateTimes = getRuntimeContext().getListState(updateDescriptor)
+ }
+
+
+ @throws[Exception]
override def processKey(
key: Int,
ctx: Context,
- out: Collector[Keyedstate]) throws Exception {
+ out: Collector[Keyedstate]): Unit = {
- val data = KeyedState(key, state.value())
- out.collect(data);
+ val data = KeyedState(key, state.value(), updateTimes.get.asScala.toList)
+ out.collect(data)
}
}
-
-val keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction());
{% endhighlight %}
-{% panel **Note:** When using a `KeyedStateReaderFunction` all state descriptors must be registered eagerly inside of open. Any attempt to call `RuntimeContext#getState`, `RuntimeContext#getListState`, or `RuntimeContext#getMapState` will result in a `RuntimeException`. %}
+Along with reading registered state values, each key has access to a `Context` with metadata such as registered event time and processing time timers.
+
+{% panel **Note:** When using a `KeyedStateReaderFunction`, all state descriptors must be registered eagerly inside of open. Any attempt to call a `RuntimeContext#get*State` will result in a `RuntimeException`. %}
## Writing New Savepoints
-State writers are based around the abstraction of `Savepoint`, where one `Savepoint` may have many operators and the state for any particular operator is created using a `BootstrapTransformation`.
+`Savepoint`'s may also be written, which allows such use cases as bootstrapping state based on historical data.
+Each savepoint is made up of one or more `BootstrapTransformation`'s (explained below), each of which defines the state for an individual operator.
-A `BootstrapTransformation` starts with a `DataSet` containing the values that are to be written into state.
-The transformation may be optionally `keyed` depending on whether or not you are writing keyed or operator state.
-Finally a bootstrap function is applied depending to the transformation; Flink supplies `KeyedStateBootstrapFunction` for writing keyed state, `StateBootstrapFunction` for writing non keyed state, and `BroadcastStateBootstrapFunction` for writing broadcast state.
+
+
+The [UIDs]({{ site.baseurl}}/ops/state/savepoints.html#assigning-operator-ids) associated with each operator must match one to one with the UIDs assigned to the operators in your `DataStream` application; these are how Flink knows what state maps to which operator.
+
+### Operator State
+
+Simple operator state, using `CheckpointedFunction`, can be created using the `StateBootstrapFunction`.
+
+
+
+### Broadcast State
+
+[BroadcastState]({{ site.baseurl }} /dev/stream/state/broadcast_state.html) can be written using a `BroadcastStateBootstrapFunction`. Similar to broadcast state in the `DataStream` API, the full state must fit in memory.
{% highlight java %}
-public class Account {
+public class CurrencyRate {
+ public String currency;
+
+ public Double rate;
+}
+
+public class CurrencyBootstrapFunction extends BroadcastStateBootstrapFunction {
+
+ public static final MapStateDescriptor descriptor =
+ new MapStateDescriptor<>("currency-rates", Types.STRING, Types.DOUBLE);
+
+ @Override
+ public void processElement(CurrencyRate value, Context ctx) throws Exception {
+ ctx.getBroadcastState(descriptor).put(value.currency, value.rate);
+ }
+}
+
+DataSet currencyDataSet = bEnv.fromCollection(
+ new CurrencyRate("USD", 1.0), new CurrencyRate("EUR", 1.3));
+
+BootstrapTransformation broadcastTransformation = OperatorTransformation
+ .bootstrapWith(currencyDataSet)
+ .transform(new CurrencyBootstrapFunction());
+{% endhighlight %}
+
+
+### Keyed State
+
+Keyed state for `ProcessFunction`'s and other `RichFunction` types can be written using a `KeyedStateBootstrapFunction`.
+
+
+
+{% highlight java %}
+public class Account {
public int id;
public double amount;
@@ -382,12 +556,13 @@ BootstrapTransformation transformation = OperatorTransformation
{% highlight scala %}
case class Account(id: Int, amount: Double, timestamp: Long)
-
+
class AccountBootstrapper extends KeyedStateBootstrapFunction[Integer, Account] {
var state: ValueState[Double]
+ @throws[Exception]
override def open(parameters: Configuration): Unit = {
- val descriptor = new ValueStateDescriptor[Double]("total",Types.DOUBLE)
+ val descriptor = new ValueStateDescriptor("total",Types.DOUBLE)
state = getRuntimeContext().getState(descriptor)
}
@@ -403,47 +578,27 @@ val accountDataSet = bEnv.fromCollection(accounts)
val transformation = OperatorTransformation
.bootstrapWith(accountDataSet)
- .keyBy(acc -> acc.id)
- .transform(new AccountBootstrapper())
+ .keyBy(acc => acc.id)
+ .transform(new AccountBootstrapper)
{% endhighlight %}
The `KeyedStateBootstrapFunction` supports setting event time and processing time timers.
The timers will not fire inside the bootstrap function and only become active once restored within a `DataStream` application.
-If a processing time timer is set but the state is not restored until after that time has passed, the timer will fire immediatly upon start.
+If a processing time timer is set but the state is not restored until after that time has passed, the timer will fire immediately upon start.
-Once one or more transformations have been created they may be combined into a single `Savepoint`.
-`Savepoint`'s are created using a state backend and max parallelism, they may contain any number of operators.
+Attention If your bootstrap function creates timers, the state can only be restored using one of the [process]({{ site.baseurl }}/dev/stream/operators/process_function.html) type functions.
-
-
-Besides creating a savepoint from scratch, you can base on off an existing savepoint such as when bootstrapping a single new operator for an existing job.
+## Modifying Savepoints
+
+Besides creating a savepoint from scratch, you can base one off an existing savepoint such as when bootstrapping a single new operator for an existing job.
{% highlight scala %}
Savepoint
- .load(backend, oldPath)
+ .load(bEnv, new MemoryStateBackend, oldPath)
.withOperator("uid", transformation)
.write(newPath)
{% endhighlight %}
diff --git a/docs/dev/stream/experimental.md b/docs/dev/stream/experimental.md
index f0a78a539126fb8f2bc466a5e44c5301d61c0105..a86eb6d09951b956a205fe53caa7231cc9820487 100644
--- a/docs/dev/stream/experimental.md
+++ b/docs/dev/stream/experimental.md
@@ -43,10 +43,10 @@ makes the second job embarrassingly parallel, which can be helpful for a fine-gr
This re-interpretation functionality is exposed through `DataStreamUtils`:
{% highlight java %}
- static KeyedStream reinterpretAsKeyedStream(
- DataStream stream,
- KeySelector keySelector,
- TypeInformation typeInfo)
+static KeyedStream reinterpretAsKeyedStream(
+ DataStream stream,
+ KeySelector keySelector,
+ TypeInformation typeInfo)
{% endhighlight %}
Given a base stream, a key selector, and type information,
@@ -57,25 +57,25 @@ Code example:
{% highlight java %}
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- DataStreamSource source = ...
- DataStreamUtils.reinterpretAsKeyedStream(source, (in) -> in, TypeInformation.of(Integer.class))
- .timeWindow(Time.seconds(1))
- .reduce((a, b) -> a + b)
- .addSink(new DiscardingSink<>());
- env.execute();
+StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+DataStreamSource source = ...
+DataStreamUtils.reinterpretAsKeyedStream(source, (in) -> in, TypeInformation.of(Integer.class))
+ .timeWindow(Time.seconds(1))
+ .reduce((a, b) -> a + b)
+ .addSink(new DiscardingSink<>());
+env.execute();
{% endhighlight %}
{% highlight scala %}
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setParallelism(1)
- val source = ...
- new DataStreamUtils(source).reinterpretAsKeyedStream((in) => in)
- .timeWindow(Time.seconds(1))
- .reduce((a, b) => a + b)
- .addSink(new DiscardingSink[Int])
- env.execute()
+val env = StreamExecutionEnvironment.getExecutionEnvironment
+env.setParallelism(1)
+val source = ...
+new DataStreamUtils(source).reinterpretAsKeyedStream((in) => in)
+ .timeWindow(Time.seconds(1))
+ .reduce((a, b) => a + b)
+ .addSink(new DiscardingSink[Int])
+env.execute()
{% endhighlight %}
{% top %}
diff --git a/docs/dev/stream/experimental.zh.md b/docs/dev/stream/experimental.zh.md
index 5b42ef994be2c117902d73f6c3fe0f1a06d3ac85..11662a99b7e0c72ed81514664609bd19fe8c0174 100644
--- a/docs/dev/stream/experimental.zh.md
+++ b/docs/dev/stream/experimental.zh.md
@@ -43,10 +43,10 @@ makes the second job embarrassingly parallel, which can be helpful for a fine-gr
This re-interpretation functionality is exposed through `DataStreamUtils`:
{% highlight java %}
- static KeyedStream reinterpretAsKeyedStream(
- DataStream stream,
- KeySelector keySelector,
- TypeInformation typeInfo)
+static KeyedStream reinterpretAsKeyedStream(
+ DataStream stream,
+ KeySelector keySelector,
+ TypeInformation typeInfo)
{% endhighlight %}
Given a base stream, a key selector, and type information,
@@ -57,25 +57,25 @@ Code example:
{% highlight java %}
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- DataStreamSource source = ...
- DataStreamUtils.reinterpretAsKeyedStream(source, (in) -> in, TypeInformation.of(Integer.class))
- .timeWindow(Time.seconds(1))
- .reduce((a, b) -> a + b)
- .addSink(new DiscardingSink<>());
- env.execute();
+StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+DataStreamSource source = ...
+DataStreamUtils.reinterpretAsKeyedStream(source, (in) -> in, TypeInformation.of(Integer.class))
+ .timeWindow(Time.seconds(1))
+ .reduce((a, b) -> a + b)
+ .addSink(new DiscardingSink<>());
+env.execute();
{% endhighlight %}
{% highlight scala %}
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setParallelism(1)
- val source = ...
- new DataStreamUtils(source).reinterpretAsKeyedStream((in) => in)
- .timeWindow(Time.seconds(1))
- .reduce((a, b) => a + b)
- .addSink(new DiscardingSink[Int])
- env.execute()
+val env = StreamExecutionEnvironment.getExecutionEnvironment
+env.setParallelism(1)
+val source = ...
+new DataStreamUtils(source).reinterpretAsKeyedStream((in) => in)
+ .timeWindow(Time.seconds(1))
+ .reduce((a, b) => a + b)
+ .addSink(new DiscardingSink[Int])
+env.execute()
{% endhighlight %}
{% top %}
diff --git a/docs/dev/stream/operators/asyncio.md b/docs/dev/stream/operators/asyncio.md
index c9a9b2cb6e23f2daabab826bcc552699e767eb3a..ba2d30b8680591e478c2dc876869f21aabe8a59b 100644
--- a/docs/dev/stream/operators/asyncio.md
+++ b/docs/dev/stream/operators/asyncio.md
@@ -227,7 +227,7 @@ asynchronous I/O operator. That means concretely the following for the two order
That means that in the presence of watermarks, the *unordered* mode introduces some of the same latency and management
overhead as the *ordered* mode does. The amount of that overhead depends on the watermark frequency.
- - **Ordered**: Order of watermarks an records is preserved, just like order between records is preserved. There is no
+ - **Ordered**: Order of watermarks and records is preserved, just like order between records is preserved. There is no
significant change in overhead, compared to working with *processing time*.
Please recall that *Ingestion Time* is a special case of *event time* with automatically generated watermarks that
diff --git a/docs/dev/stream/operators/index.md b/docs/dev/stream/operators/index.md
index b89fa898fe7940f49db36af6acd08684e9700d9e..685bd33505bad0b13252014363d414979c3fcb09 100644
--- a/docs/dev/stream/operators/index.md
+++ b/docs/dev/stream/operators/index.md
@@ -52,7 +52,7 @@ partitioning after applying those as well as insights into Flink's operator chai
Map DataStream → DataStream
Takes one element and produces one element. A map function that doubles the values of the input stream:
Logically partitions a stream into disjoint partitions. All records with the same key are assigned to the same partition. Internally, keyBy() is implemented with hash partitioning. There are different ways to specify keys.
This transformation returns a KeyedStream, which is, among other things, required to use keyed state.
- {% highlight java %}
+{% highlight java %}
dataStream.keyBy("someKey") // Key by field "someKey"
dataStream.keyBy(0) // Key by the first element of a Tuple
- {% endhighlight %}
+{% endhighlight %}
Attention
A type cannot be a key if:
@@ -126,7 +126,7 @@ dataStream.keyBy(0) // Key by the first element of a Tuple
A reduce function that creates a stream of partial sums:
A fold function that, when applied on the sequence (1,2,3,4,5),
emits the sequence "start-1", "start-1-2", "start-1-2-3", ...
- {% highlight java %}
+{% highlight java %}
DataStream result =
keyedStream.fold("start", new FoldFunction() {
@Override
@@ -156,7 +156,7 @@ DataStream result =
return current + "-" + value;
}
});
- {% endhighlight %}
+{% endhighlight %}
@@ -166,7 +166,7 @@ DataStream result =
Rolling aggregations on a keyed data stream. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Windows can be defined on already partitioned KeyedStreams. Windows group the data in each
key according to some characteristic (e.g., the data that arrived within the last 5 seconds).
See windows for a complete description of windows.
- {% highlight java %}
+{% highlight java %}
dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))); // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -200,9 +200,9 @@ dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))); // Las
See windows for a complete description of windows.
WARNING: This is in many cases a non-parallel transformation. All records will be
gathered in one task for the windowAll operator.
- {% highlight java %}
+{% highlight java %}
dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))); // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -210,7 +210,7 @@ dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))); // Last 5 se
Applies a general function to the window as a whole. Below is a function that manually sums the elements of a window.
Note: If you are using a windowAll transformation, you need to use an AllWindowFunction instead.
@@ -258,13 +258,13 @@ windowedStream.reduce (new ReduceFunction>() {
Applies a functional fold function to the window and returns the folded value.
The example function, when applied on the sequence (1,2,3,4,5),
folds the sequence into the string "start-1-2-3-4-5":
@@ -273,7 +273,7 @@ windowedStream.fold("start", new FoldFunction() {
Aggregates the contents of a window. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Union of two or more data streams creating a new stream containing all the elements from all the streams. Note: If you union a data stream
with itself you will get each element twice in the resulting stream.
Join two elements e1 and e2 of two keyed streams with a common key over a given time interval, so that e1.timestamp + lowerBound <= e2.timestamp <= e1.timestamp + upperBound
- {% highlight java %}
+{% highlight java %}
// this will join the two streams so that
// key1 == key2 && leftTs - 2 < rightTs < leftTs + 2
keyedStream.intervalJoin(otherKeyedStream)
@@ -321,19 +321,19 @@ keyedStream.intervalJoin(otherKeyedStream)
.upperBoundExclusive(true) // optional
.lowerBoundExclusive(true) // optional
.process(new IntervalJoinFunction() {...});
- {% endhighlight %}
+{% endhighlight %}
Window CoGroup DataStream,DataStream → DataStream
Cogroups two data streams on a given key and a common window.
Select one or more streams from a split stream.
- {% highlight java %}
+{% highlight java %}
SplitStream split;
DataStream even = split.select("even");
DataStream odd = split.select("odd");
DataStream all = split.select("even","odd");
- {% endhighlight %}
+{% endhighlight %}
@@ -429,7 +429,7 @@ DataStream all = split.select("even","odd");
the iteration body continuously. Elements that are greater than 0 are sent back
to the feedback channel, and the rest of the elements are forwarded downstream.
See iterations for a complete description.
- {% highlight java %}
+{% highlight java %}
IterativeStream iteration = initialStream.iterate();
DataStream iterationBody = iteration.map (/*do something*/);
DataStream feedback = iterationBody.filter(new FilterFunction(){
@@ -445,7 +445,7 @@ DataStream output = iterationBody.filter(new FilterFunction(){
return value <= 0;
}
});
- {% endhighlight %}
+{% endhighlight %}
@@ -455,9 +455,9 @@ DataStream output = iterationBody.filter(new FilterFunction(){
Extracts timestamps from records in order to work with windows
that use event time semantics. See Event Time.
- {% highlight java %}
+{% highlight java %}
stream.assignTimestamps (new TimeStampExtractor() {...});
- {% endhighlight %}
+{% endhighlight %}
@@ -482,9 +482,9 @@ stream.assignTimestamps (new TimeStampExtractor() {...});
Map DataStream → DataStream
Takes one element and produces one element. A map function that doubles the values of the input stream:
Logically partitions a stream into disjoint partitions, each partition containing elements of the same key.
Internally, this is implemented with hash partitioning. See keys on how to specify keys.
This transformation returns a KeyedStream.
- {% highlight scala %}
+{% highlight scala %}
dataStream.keyBy("someKey") // Key by field "someKey"
dataStream.keyBy(0) // Key by the first element of a Tuple
- {% endhighlight %}
+{% endhighlight %}
@@ -528,9 +528,9 @@ dataStream.keyBy(0) // Key by the first element of a Tuple
A reduce function that creates a stream of partial sums:
- {% highlight scala %}
+{% highlight scala %}
keyedStream.reduce { _ + _ }
- {% endhighlight %}
+{% endhighlight %}
Rolling aggregations on a keyed data stream. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Windows can be defined on already partitioned KeyedStreams. Windows group the data in each
key according to some characteristic (e.g., the data that arrived within the last 5 seconds).
See windows for a description of windows.
- {% highlight scala %}
+{% highlight scala %}
dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))) // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -591,9 +591,9 @@ dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))) // Last
See windows for a complete description of windows.
WARNING: This is in many cases a non-parallel transformation. All records will be
gathered in one task for the windowAll operator.
- {% highlight scala %}
+{% highlight scala %}
dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))) // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -601,22 +601,22 @@ dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))) // Last 5 sec
Applies a general function to the window as a whole. Below is a function that manually sums the elements of a window.
Note: If you are using a windowAll transformation, you need to use an AllWindowFunction instead.
Applies a functional fold function to the window and returns the folded value.
The example function, when applied on the sequence (1,2,3,4,5),
folds the sequence into the string "start-1-2-3-4-5":
@@ -637,7 +637,7 @@ val result: DataStream[String] =
Aggregates the contents of a window. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Union of two or more data streams creating a new stream containing all the elements from all the streams. Note: If you union a data stream
with itself you will get each element twice in the resulting stream.
Split the stream into two or more streams according to some criterion.
- {% highlight scala %}
+{% highlight scala %}
val split = someDataStream.split(
(num: Int) =>
(num % 2) match {
@@ -727,7 +727,7 @@ val split = someDataStream.split(
case 1 => List("odd")
}
)
- {% endhighlight %}
+{% endhighlight %}
@@ -736,12 +736,12 @@ val split = someDataStream.split(
Select one or more streams from a split stream.
- {% highlight scala %}
+{% highlight scala %}
val even = split select "even"
val odd = split select "odd"
val all = split.select("even","odd")
- {% endhighlight %}
+{% endhighlight %}
@@ -755,14 +755,14 @@ val all = split.select("even","odd")
the iteration body continuously. Elements that are greater than 0 are sent back
to the feedback channel, and the rest of the elements are forwarded downstream.
See iterations for a complete description.
- {% highlight java %}
+{% highlight java %}
initialStream.iterate {
iteration => {
val iterationBody = iteration.map {/*do something*/}
(iterationBody.filter(_ > 0), iterationBody.filter(_ <= 0))
}
}
- {% endhighlight %}
+{% endhighlight %}
@@ -773,9 +773,9 @@ initialStream.iterate {
Extracts timestamps from records in order to work with windows
that use event time semantics.
See Event Time.
- {% highlight scala %}
+{% highlight scala %}
stream.assignTimestamps { timestampExtractor }
- {% endhighlight %}
+{% endhighlight %}
@@ -852,10 +852,10 @@ via the following functions.
Uses a user-defined Partitioner to select the target task for each element.
- {% highlight java %}
+{% highlight java %}
dataStream.partitionCustom(partitioner, "someKey");
dataStream.partitionCustom(partitioner, 0);
- {% endhighlight %}
+{% endhighlight %}
Partitions elements randomly according to a uniform distribution.
- {% highlight java %}
+{% highlight java %}
dataStream.shuffle();
- {% endhighlight %}
+{% endhighlight %}
@@ -876,9 +876,9 @@ dataStream.shuffle();
Partitions elements round-robin, creating equal load per partition. Useful for performance
optimization in the presence of data skew.
- {% highlight java %}
+{% highlight java %}
dataStream.rebalance();
- {% endhighlight %}
+{% endhighlight %}
Partitions elements randomly according to a uniform distribution.
- {% highlight scala %}
+{% highlight scala %}
dataStream.shuffle()
- {% endhighlight %}
+{% endhighlight %}
@@ -983,9 +983,9 @@ dataStream.shuffle()
Partitions elements round-robin, creating equal load per partition. Useful for performance
optimization in the presence of data skew.
- {% highlight scala %}
+{% highlight scala %}
dataStream.rebalance()
- {% endhighlight %}
+{% endhighlight %}
@@ -1016,7 +1016,7 @@ dataStream.rebalance()
downstream operations will have a differing number of inputs from upstream operations.
-
+
Please see this figure for a visualization of the connection pattern in the above
example:
Broadcasts elements to every partition.
- {% highlight scala %}
+{% highlight scala %}
dataStream.broadcast()
- {% endhighlight %}
+{% endhighlight %}
diff --git a/docs/dev/stream/operators/index.zh.md b/docs/dev/stream/operators/index.zh.md
index 1aca1dabb11132babcb987ee683c9a74e090cfc3..d5581f09fada105bfe5a6f83a0cecd6a24cc5251 100644
--- a/docs/dev/stream/operators/index.zh.md
+++ b/docs/dev/stream/operators/index.zh.md
@@ -52,7 +52,7 @@ partitioning after applying those as well as insights into Flink's operator chai
Map DataStream → DataStream
Takes one element and produces one element. A map function that doubles the values of the input stream:
Logically partitions a stream into disjoint partitions. All records with the same key are assigned to the same partition. Internally, keyBy() is implemented with hash partitioning. There are different ways to specify keys.
This transformation returns a KeyedStream, which is, among other things, required to use keyed state.
- {% highlight java %}
+{% highlight java %}
dataStream.keyBy("someKey") // Key by field "someKey"
dataStream.keyBy(0) // Key by the first element of a Tuple
- {% endhighlight %}
+{% endhighlight %}
Attention
A type cannot be a key if:
@@ -126,7 +126,7 @@ dataStream.keyBy(0) // Key by the first element of a Tuple
A reduce function that creates a stream of partial sums:
A fold function that, when applied on the sequence (1,2,3,4,5),
emits the sequence "start-1", "start-1-2", "start-1-2-3", ...
- {% highlight java %}
+{% highlight java %}
DataStream result =
keyedStream.fold("start", new FoldFunction() {
@Override
@@ -156,7 +156,7 @@ DataStream result =
return current + "-" + value;
}
});
- {% endhighlight %}
+{% endhighlight %}
@@ -166,7 +166,7 @@ DataStream result =
Rolling aggregations on a keyed data stream. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Windows can be defined on already partitioned KeyedStreams. Windows group the data in each
key according to some characteristic (e.g., the data that arrived within the last 5 seconds).
See windows for a complete description of windows.
- {% highlight java %}
+{% highlight java %}
dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))); // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -200,9 +200,9 @@ dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))); // Las
See windows for a complete description of windows.
WARNING: This is in many cases a non-parallel transformation. All records will be
gathered in one task for the windowAll operator.
- {% highlight java %}
+{% highlight java %}
dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))); // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -210,7 +210,7 @@ dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))); // Last 5 se
Applies a general function to the window as a whole. Below is a function that manually sums the elements of a window.
Note: If you are using a windowAll transformation, you need to use an AllWindowFunction instead.
@@ -258,13 +258,13 @@ windowedStream.reduce (new ReduceFunction>() {
Applies a functional fold function to the window and returns the folded value.
The example function, when applied on the sequence (1,2,3,4,5),
folds the sequence into the string "start-1-2-3-4-5":
@@ -273,7 +273,7 @@ windowedStream.fold("start", new FoldFunction() {
Aggregates the contents of a window. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Union of two or more data streams creating a new stream containing all the elements from all the streams. Note: If you union a data stream
with itself you will get each element twice in the resulting stream.
Join two elements e1 and e2 of two keyed streams with a common key over a given time interval, so that e1.timestamp + lowerBound <= e2.timestamp <= e1.timestamp + upperBound
- {% highlight java %}
+{% highlight java %}
// this will join the two streams so that
// key1 == key2 && leftTs - 2 < rightTs < leftTs + 2
keyedStream.intervalJoin(otherKeyedStream)
@@ -321,19 +321,19 @@ keyedStream.intervalJoin(otherKeyedStream)
.upperBoundExclusive(true) // optional
.lowerBoundExclusive(true) // optional
.process(new IntervalJoinFunction() {...});
- {% endhighlight %}
+{% endhighlight %}
Window CoGroup DataStream,DataStream → DataStream
Cogroups two data streams on a given key and a common window.
Select one or more streams from a split stream.
- {% highlight java %}
+{% highlight java %}
SplitStream split;
DataStream even = split.select("even");
DataStream odd = split.select("odd");
DataStream all = split.select("even","odd");
- {% endhighlight %}
+{% endhighlight %}
@@ -429,7 +429,7 @@ DataStream all = split.select("even","odd");
the iteration body continuously. Elements that are greater than 0 are sent back
to the feedback channel, and the rest of the elements are forwarded downstream.
See iterations for a complete description.
- {% highlight java %}
+{% highlight java %}
IterativeStream iteration = initialStream.iterate();
DataStream iterationBody = iteration.map (/*do something*/);
DataStream feedback = iterationBody.filter(new FilterFunction(){
@@ -445,7 +445,7 @@ DataStream output = iterationBody.filter(new FilterFunction(){
return value <= 0;
}
});
- {% endhighlight %}
+{% endhighlight %}
@@ -455,9 +455,9 @@ DataStream output = iterationBody.filter(new FilterFunction(){
Extracts timestamps from records in order to work with windows
that use event time semantics. See Event Time.
- {% highlight java %}
+{% highlight java %}
stream.assignTimestamps (new TimeStampExtractor() {...});
- {% endhighlight %}
+{% endhighlight %}
@@ -482,9 +482,9 @@ stream.assignTimestamps (new TimeStampExtractor() {...});
Map DataStream → DataStream
Takes one element and produces one element. A map function that doubles the values of the input stream:
Logically partitions a stream into disjoint partitions, each partition containing elements of the same key.
Internally, this is implemented with hash partitioning. See keys on how to specify keys.
This transformation returns a KeyedStream.
- {% highlight scala %}
+{% highlight scala %}
dataStream.keyBy("someKey") // Key by field "someKey"
dataStream.keyBy(0) // Key by the first element of a Tuple
- {% endhighlight %}
+{% endhighlight %}
@@ -528,9 +528,9 @@ dataStream.keyBy(0) // Key by the first element of a Tuple
A reduce function that creates a stream of partial sums:
- {% highlight scala %}
+{% highlight scala %}
keyedStream.reduce { _ + _ }
- {% endhighlight %}
+{% endhighlight %}
Rolling aggregations on a keyed data stream. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Windows can be defined on already partitioned KeyedStreams. Windows group the data in each
key according to some characteristic (e.g., the data that arrived within the last 5 seconds).
See windows for a description of windows.
- {% highlight scala %}
+{% highlight scala %}
dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))) // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -591,9 +591,9 @@ dataStream.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))) // Last
See windows for a complete description of windows.
WARNING: This is in many cases a non-parallel transformation. All records will be
gathered in one task for the windowAll operator.
- {% highlight scala %}
+{% highlight scala %}
dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))) // Last 5 seconds of data
- {% endhighlight %}
+{% endhighlight %}
@@ -601,22 +601,22 @@ dataStream.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))) // Last 5 sec
Applies a general function to the window as a whole. Below is a function that manually sums the elements of a window.
Note: If you are using a windowAll transformation, you need to use an AllWindowFunction instead.
Applies a functional fold function to the window and returns the folded value.
The example function, when applied on the sequence (1,2,3,4,5),
folds the sequence into the string "start-1-2-3-4-5":
@@ -637,7 +637,7 @@ val result: DataStream[String] =
Aggregates the contents of a window. The difference between min
and minBy is that min returns the minimum value, whereas minBy returns
the element that has the minimum value in this field (same for max and maxBy).
Union of two or more data streams creating a new stream containing all the elements from all the streams. Note: If you union a data stream
with itself you will get each element twice in the resulting stream.
Split the stream into two or more streams according to some criterion.
- {% highlight scala %}
+{% highlight scala %}
val split = someDataStream.split(
(num: Int) =>
(num % 2) match {
@@ -727,7 +727,7 @@ val split = someDataStream.split(
case 1 => List("odd")
}
)
- {% endhighlight %}
+{% endhighlight %}
@@ -736,12 +736,12 @@ val split = someDataStream.split(
Select one or more streams from a split stream.
- {% highlight scala %}
+{% highlight scala %}
val even = split select "even"
val odd = split select "odd"
val all = split.select("even","odd")
- {% endhighlight %}
+{% endhighlight %}
@@ -755,14 +755,14 @@ val all = split.select("even","odd")
the iteration body continuously. Elements that are greater than 0 are sent back
to the feedback channel, and the rest of the elements are forwarded downstream.
See iterations for a complete description.
- {% highlight java %}
+{% highlight java %}
initialStream.iterate {
iteration => {
val iterationBody = iteration.map {/*do something*/}
(iterationBody.filter(_ > 0), iterationBody.filter(_ <= 0))
}
}
- {% endhighlight %}
+{% endhighlight %}
@@ -773,9 +773,9 @@ initialStream.iterate {
Extracts timestamps from records in order to work with windows
that use event time semantics.
See Event Time.
- {% highlight scala %}
+{% highlight scala %}
stream.assignTimestamps { timestampExtractor }
- {% endhighlight %}
+{% endhighlight %}
@@ -852,10 +852,10 @@ via the following functions.
Uses a user-defined Partitioner to select the target task for each element.
- {% highlight java %}
+{% highlight java %}
dataStream.partitionCustom(partitioner, "someKey");
dataStream.partitionCustom(partitioner, 0);
- {% endhighlight %}
+{% endhighlight %}
Partitions elements randomly according to a uniform distribution.
- {% highlight java %}
+{% highlight java %}
dataStream.shuffle();
- {% endhighlight %}
+{% endhighlight %}
@@ -876,9 +876,9 @@ dataStream.shuffle();
Partitions elements round-robin, creating equal load per partition. Useful for performance
optimization in the presence of data skew.
- {% highlight java %}
+{% highlight java %}
dataStream.rebalance();
- {% endhighlight %}
+{% endhighlight %}
Partitions elements randomly according to a uniform distribution.
- {% highlight scala %}
+{% highlight scala %}
dataStream.shuffle()
- {% endhighlight %}
+{% endhighlight %}
@@ -983,9 +983,9 @@ dataStream.shuffle()
Partitions elements round-robin, creating equal load per partition. Useful for performance
optimization in the presence of data skew.
- {% highlight scala %}
+{% highlight scala %}
dataStream.rebalance()
- {% endhighlight %}
+{% endhighlight %}
@@ -1016,7 +1016,7 @@ dataStream.rebalance()
downstream operations will have a differing number of inputs from upstream operations.
-
+
Please see this figure for a visualization of the connection pattern in the above
example:
Broadcasts elements to every partition.
- {% highlight scala %}
+{% highlight scala %}
dataStream.broadcast()
- {% endhighlight %}
+{% endhighlight %}
diff --git a/docs/dev/stream/operators/process_function.md b/docs/dev/stream/operators/process_function.md
index 79212311ad9c558a8caf281ba191a78bb053c0be..ef51b84758d4df736517203501f6fc55be40139f 100644
--- a/docs/dev/stream/operators/process_function.md
+++ b/docs/dev/stream/operators/process_function.md
@@ -1,5 +1,5 @@
---
-title: "Process Function (Low-level Operations)"
+title: "Process Function"
nav-title: "Process Function"
nav-parent_id: streaming_operators
nav-pos: 35
@@ -44,8 +44,7 @@ For fault-tolerant state, the `ProcessFunction` gives access to Flink's [keyed s
The timers allow applications to react to changes in processing time and in [event time]({{ site.baseurl }}/dev/event_time.html).
Every call to the function `processElement(...)` gets a `Context` object which gives access to the element's
event time timestamp, and to the *TimerService*. The `TimerService` can be used to register callbacks for future
-event-/processing-time instants. When a timer's particular time is reached, the `onTimer(...)` method is
-called. During that call, all states are again scoped to the key with which the timer was created, allowing
+event-/processing-time instants. With event-time timers, the `onTimer(...)` method is called when the current watermark is advanced up to or beyond the timestamp of the timer, while with processing-time timers, `onTimer(...)` is called when wall clock time reaches the specified time. During that call, all states are again scoped to the key with which the timer was created, allowing
timers to manipulate keyed state.
Note If you want to access keyed state and timers you have
diff --git a/docs/dev/stream/operators/process_function.zh.md b/docs/dev/stream/operators/process_function.zh.md
index 41c339c30fe662a283fec9ab530e04e068680561..240da52abcd31c889fd49f89da952a70a5003cab 100644
--- a/docs/dev/stream/operators/process_function.zh.md
+++ b/docs/dev/stream/operators/process_function.zh.md
@@ -1,5 +1,5 @@
---
-title: "Process Function (Low-level Operations)"
+title: "Process Function"
nav-title: "Process Function"
nav-parent_id: streaming_operators
nav-pos: 35
@@ -44,8 +44,7 @@ For fault-tolerant state, the `ProcessFunction` gives access to Flink's [keyed s
The timers allow applications to react to changes in processing time and in [event time]({{ site.baseurl }}/dev/event_time.html).
Every call to the function `processElement(...)` gets a `Context` object which gives access to the element's
event time timestamp, and to the *TimerService*. The `TimerService` can be used to register callbacks for future
-event-/processing-time instants. When a timer's particular time is reached, the `onTimer(...)` method is
-called. During that call, all states are again scoped to the key with which the timer was created, allowing
+event-/processing-time instants. With event-time timers, the `onTimer(...)` method is called when the current watermark is advanced up to or beyond the timestamp of the timer, while with processing-time timers, `onTimer(...)` is called when wall clock time reaches the specified time. During that call, all states are again scoped to the key with which the timer was created, allowing
timers to manipulate keyed state.
Note If you want to access keyed state and timers you have
diff --git a/docs/dev/stream/operators/windows.md b/docs/dev/stream/operators/windows.md
index d317b2ee84fa922c64afc986f78439ef37d18dea..a71177d0da029afc792406c3d384f9daf8293712 100644
--- a/docs/dev/stream/operators/windows.md
+++ b/docs/dev/stream/operators/windows.md
@@ -759,7 +759,7 @@ input
class MyProcessWindowFunction extends ProcessWindowFunction[(String, Long), String, String, TimeWindow] {
- def process(key: String, context: Context, input: Iterable[(String, Long)], out: Collector[String]): () = {
+ def process(key: String, context: Context, input: Iterable[(String, Long)], out: Collector[String]) = {
var count = 0L
for (in <- input) {
count = count + 1
@@ -938,7 +938,7 @@ class AverageAggregate extends AggregateFunction[(String, Long), (Long, Long), D
class MyProcessWindowFunction extends ProcessWindowFunction[Double, (String, Double), String, TimeWindow] {
- def process(key: String, context: Context, averages: Iterable[Double], out: Collector[(String, Double)]): () = {
+ def process(key: String, context: Context, averages: Iterable[Double], out: Collector[(String, Double)]) = {
val average = averages.iterator.next()
out.collect((key, average))
}
diff --git a/docs/dev/stream/operators/windows.zh.md b/docs/dev/stream/operators/windows.zh.md
index 45ef36f12ad47bcf1be958cac992b2bdd892dcbc..72269a45596d99c11164105970ba9d08f181b9e9 100644
--- a/docs/dev/stream/operators/windows.zh.md
+++ b/docs/dev/stream/operators/windows.zh.md
@@ -759,7 +759,7 @@ input
class MyProcessWindowFunction extends ProcessWindowFunction[(String, Long), String, String, TimeWindow] {
- def process(key: String, context: Context, input: Iterable[(String, Long)], out: Collector[String]): () = {
+ def process(key: String, context: Context, input: Iterable[(String, Long)], out: Collector[String]) = {
var count = 0L
for (in <- input) {
count = count + 1
@@ -938,7 +938,7 @@ class AverageAggregate extends AggregateFunction[(String, Long), (Long, Long), D
class MyProcessWindowFunction extends ProcessWindowFunction[Double, (String, Double), String, TimeWindow] {
- def process(key: String, context: Context, averages: Iterable[Double], out: Collector[(String, Double)]): () = {
+ def process(key: String, context: Context, averages: Iterable[Double], out: Collector[(String, Double)]) = {
val average = averages.iterator.next()
out.collect((key, average))
}
diff --git a/docs/dev/stream/state/broadcast_state.md b/docs/dev/stream/state/broadcast_state.md
index 73b389aca27d5c951cb803ed3306ccd971b7e580..ba126188323f743089e84a29684153dd0100facb 100644
--- a/docs/dev/stream/state/broadcast_state.md
+++ b/docs/dev/stream/state/broadcast_state.md
@@ -25,18 +25,9 @@ under the License.
* ToC
{:toc}
-[Working with State](state.html) describes operator state which upon restore is either evenly distributed among the
-parallel tasks of an operator, or unioned, with the whole state being used to initialize the restored parallel tasks.
-
-A third type of supported *operator state* is the *Broadcast State*. Broadcast state was introduced to support use cases
-where some data coming from one stream is required to be broadcasted to all downstream tasks, where it is stored locally
-and is used to process all incoming elements on the other stream. As an example where broadcast state can emerge as a
-natural fit, one can imagine a low-throughput stream containing a set of rules which we want to evaluate against all
-elements coming from another stream. Having the above type of use cases in mind, broadcast state differs from the rest
-of operator states in that:
- 1. it has a map format,
- 2. it is only available to specific operators that have as inputs a *broadcasted* stream and a *non-broadcasted* one, and
- 3. such an operator can have *multiple broadcast states* with different names.
+In this section you will learn about how to use broadcast state in practise. Please refer to [Stateful Stream
+Processing]({{site.baseurl}}{% link concepts/stateful-stream-processing.md %})
+to learn about the concepts behind stateful stream processing.
## Provided APIs
diff --git a/docs/dev/stream/state/checkpointing.zh.md b/docs/dev/stream/state/checkpointing.zh.md
index c193fc37d71ed5639cbbe32a75da6abf2b3c7da0..d4aa989523a6803660d38fd19253daa99f511c65 100644
--- a/docs/dev/stream/state/checkpointing.zh.md
+++ b/docs/dev/stream/state/checkpointing.zh.md
@@ -25,85 +25,77 @@ under the License.
* ToC
{:toc}
-Every function and operator in Flink can be **stateful** (see [working with state](state.html) for details).
-Stateful functions store data across the processing of individual elements/events, making state a critical building block for
-any type of more elaborate operation.
+Flink 中的每个方法或算子都能够是**有状态的**（阅读 [working with state](state.html) 了解更多）。
+状态化的方法在处理单个 元素/事件 的时候存储数据，让状态成为使各个类型的算子更加精细的重要部分。
+为了让状态容错，Flink 需要为状态添加 **checkpoint（检查点）**。Checkpoint 使得 Flink 能够恢复状态和在流中的位置，从而向应用提供和无故障执行时一样的语义。
-In order to make state fault tolerant, Flink needs to **checkpoint** the state. Checkpoints allow Flink to recover state and positions
-in the streams to give the application the same semantics as a failure-free execution.
+[容错文档]({{ site.baseurl }}/zh/internals/stream_checkpointing.html) 中介绍了 Flink 流计算容错机制内部的技术原理。
-The [documentation on streaming fault tolerance]({{ site.baseurl }}/internals/stream_checkpointing.html) describes in detail the technique behind Flink's streaming fault tolerance mechanism.
+## 前提条件
-## Prerequisites
+Flink 的 checkpoint 机制会和持久化存储进行交互，读写流与状态。一般需要：
-Flink's checkpointing mechanism interacts with durable storage for streams and state. In general, it requires:
+ - 一个能够回放一段时间内数据的持久化数据源，例如持久化消息队列（例如 Apache Kafka、RabbitMQ、 Amazon Kinesis、 Google PubSub 等）或文件系统（例如 HDFS、 S3、 GFS、 NFS、 Ceph 等）。
+ - 存放状态的持久化存储，通常为分布式文件系统（比如 HDFS、 S3、 GFS、 NFS、 Ceph 等）。
- - A *persistent* (or *durable*) data source that can replay records for a certain amount of time. Examples for such sources are persistent messages queues (e.g., Apache Kafka, RabbitMQ, Amazon Kinesis, Google PubSub) or file systems (e.g., HDFS, S3, GFS, NFS, Ceph, ...).
- - A persistent storage for state, typically a distributed filesystem (e.g., HDFS, S3, GFS, NFS, Ceph, ...)
+## 开启与配置 Checkpoint
+默认情况下 checkpoint 是禁用的。通过调用 `StreamExecutionEnvironment` 的 `enableCheckpointing(n)` 来启用 checkpoint，里面的 *n* 是进行 checkpoint 的间隔，单位毫秒。
-## Enabling and Configuring Checkpointing
+Checkpoint 其他的属性包括：
-By default, checkpointing is disabled. To enable checkpointing, call `enableCheckpointing(n)` on the `StreamExecutionEnvironment`, where *n* is the checkpoint interval in milliseconds.
-
-Other parameters for checkpointing include:
-
- - *exactly-once vs. at-least-once*: You can optionally pass a mode to the `enableCheckpointing(n)` method to choose between the two guarantee levels.
- Exactly-once is preferable for most applications. At-least-once may be relevant for certain super-low-latency (consistently few milliseconds) applications.
-
- - *checkpoint timeout*: The time after which a checkpoint-in-progress is aborted, if it did not complete by then.
-
- - *minimum time between checkpoints*: To make sure that the streaming application makes a certain amount of progress between checkpoints,
- one can define how much time needs to pass between checkpoints. If this value is set for example to *5000*, the next checkpoint will be
- started no sooner than 5 seconds after the previous checkpoint completed, regardless of the checkpoint duration and the checkpoint interval.
- Note that this implies that the checkpoint interval will never be smaller than this parameter.
+ - *精确一次（exactly-once）对比至少一次（at-least-once）*：你可以选择向 `enableCheckpointing(long interval, CheckpointingMode mode)` 方法中传入一个模式来选择使用两种保证等级中的哪一种。
+ 对于大多数应用来说，精确一次是较好的选择。至少一次可能与某些延迟超低（始终只有几毫秒）的应用的关联较大。
+
+ - *checkpoint 超时*：如果 checkpoint 执行的时间超过了该配置的阈值，还在进行中的 checkpoint 操作就会被抛弃。
+
+ - *checkpoints 之间的最小时间*：该属性定义在 checkpoint 之间需要多久的时间，以确保流应用在 checkpoint 之间有足够的进展。如果值设置为了 *5000*，
+ 无论 checkpoint 持续时间与间隔是多久，在前一个 checkpoint 完成时的至少五秒后会才开始下一个 checkpoint。
- It is often easier to configure applications by defining the "time between checkpoints" than the checkpoint interval, because the "time between checkpoints"
- is not susceptible to the fact that checkpoints may sometimes take longer than on average (for example if the target storage system is temporarily slow).
-
- Note that this value also implies that the number of concurrent checkpoints is *one*.
-
- - *number of concurrent checkpoints*: By default, the system will not trigger another checkpoint while one is still in progress.
- This ensures that the topology does not spend too much time on checkpoints and not make progress with processing the streams.
- It is possible to allow for multiple overlapping checkpoints, which is interesting for pipelines that have a certain processing delay
- (for example because the functions call external services that need some time to respond) but that still want to do very frequent checkpoints
- (100s of milliseconds) to re-process very little upon failures.
-
- This option cannot be used when a minimum time between checkpoints is defined.
-
- - *externalized checkpoints*: You can configure periodic checkpoints to be persisted externally. Externalized checkpoints write their meta data out to persistent storage and are *not* automatically cleaned up when the job fails. This way, you will have a checkpoint around to resume from if your job fails. There are more details in the [deployment notes on externalized checkpoints]({{ site.baseurl }}/ops/state/checkpoints.html#externalized-checkpoints).
-
- - *fail/continue task on checkpoint errors*: This determines if a task will be failed if an error occurs in the execution of the task's checkpoint procedure. This is the default behaviour. Alternatively, when this is disabled, the task will simply decline the checkpoint to the checkpoint coordinator and continue running.
+ 往往使用“checkpoints 之间的最小时间”来配置应用会比 checkpoint 间隔容易很多，因为“checkpoints 之间的最小时间”在 checkpoint 的执行时间超过平均值时不会受到影响（例如如果目标的存储系统忽然变得很慢）。
+
+ 注意这个值也意味着并发 checkpoint 的数目是*一*。
- - *prefer checkpoint for recovery*: This determines if a job will fallback to latest checkpoint even when there are more recent savepoints available to potentially reduce recovery time.
+ - *并发 checkpoint 的数目*: 默认情况下，在上一个 checkpoint 未完成（失败或者成功）的情况下，系统不会触发另一个 checkpoint。这确保了拓扑不会在 checkpoint 上花费太多时间，从而影响正常的处理流程。
+ 不过允许多个 checkpoint 并行进行是可行的，对于有确定的处理延迟（例如某方法所调用比较耗时的外部服务），但是仍然想进行频繁的 checkpoint 去最小化故障后重跑的 pipelines 来说，是有意义的。
+
+ 该选项不能和 "checkpoints 间的最小时间"同时使用。
+
+ - *externalized checkpoints*: 你可以配置周期存储 checkpoint 到外部系统中。Externalized checkpoints 将他们的元数据写到持久化存储上并且在 job 失败的时候*不会*被自动删除。
+ 这种方式下，如果你的 job 失败，你将会有一个现有的 checkpoint 去恢复。更多的细节请看 [Externalized checkpoints 的部署文档]({{ site.baseurl }}/zh/ops/state/checkpoints.html#externalized-checkpoints)。
+
+ - *在 checkpoint 出错时使 task 失败或者继续进行 task*：他决定了在 task checkpoint 的过程中发生错误时，是否使 task 也失败，使失败是默认的行为。
+ 或者禁用它时，这个任务将会简单的把 checkpoint 错误信息报告给 checkpoint coordinator 并继续运行。
+
+ - *优先从 checkpoint 恢复（prefer checkpoint for recovery）*：该属性确定 job 是否在最新的 checkpoint 回退，即使有更近的 savepoint 可用，这可以潜在地减少恢复时间（checkpoint 恢复比 savepoint 恢复更快）。
{% highlight java %}
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-// start a checkpoint every 1000 ms
+// 每 1000ms 开始一次 checkpoint
env.enableCheckpointing(1000);
-// advanced options:
+// 高级选项：
-// set mode to exactly-once (this is the default)
+// 设置模式为精确一次 (这是默认值)
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
-// make sure 500 ms of progress happen between checkpoints
+// 确认 checkpoints 之间的时间会进行 500 ms
env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500);
-// checkpoints have to complete within one minute, or are discarded
+// Checkpoint 必须在一分钟内完成，否则就会被抛弃
env.getCheckpointConfig().setCheckpointTimeout(60000);
-// allow only one checkpoint to be in progress at the same time
+// 同一时间只允许一个 checkpoint 进行
env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
-// enable externalized checkpoints which are retained after job cancellation
+// 开启在 job 中止后仍然保留的 externalized checkpoints
env.getCheckpointConfig().enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
-// allow job recovery fallback to checkpoint when there is a more recent savepoint
+// 允许在有更近 savepoint 时回退到 checkpoint
env.getCheckpointConfig().setPreferCheckpointForRecovery(true);
{% endhighlight %}
@@ -111,24 +103,24 @@ env.getCheckpointConfig().setPreferCheckpointForRecovery(true);
{% highlight scala %}
val env = StreamExecutionEnvironment.getExecutionEnvironment()
-// start a checkpoint every 1000 ms
+// 每 1000ms 开始一次 checkpoint
env.enableCheckpointing(1000)
-// advanced options:
+// 高级选项：
-// set mode to exactly-once (this is the default)
+// 设置模式为精确一次 (这是默认值)
env.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE)
-// make sure 500 ms of progress happen between checkpoints
+// 确认 checkpoints 之间的时间会进行 500 ms
env.getCheckpointConfig.setMinPauseBetweenCheckpoints(500)
-// checkpoints have to complete within one minute, or are discarded
+// Checkpoint 必须在一分钟内完成，否则就会被抛弃
env.getCheckpointConfig.setCheckpointTimeout(60000)
-// prevent the tasks from failing if an error happens in their checkpointing, the checkpoint will just be declined.
+// 如果 task 的 checkpoint 发生错误，会阻止 task 失败，checkpoint 仅仅会被抛弃
env.getCheckpointConfig.setFailTasksOnCheckpointingErrors(false)
-// allow only one checkpoint to be in progress at the same time
+// 同一时间只允许一个 checkpoint 进行
env.getCheckpointConfig.setMaxConcurrentCheckpoints(1)
{% endhighlight %}
@@ -136,67 +128,63 @@ env.getCheckpointConfig.setMaxConcurrentCheckpoints(1)
{% highlight python %}
env = StreamExecutionEnvironment.get_execution_environment()
-# start a checkpoint every 1000 ms
+# 每 1000ms 开始一次 checkpoint
env.enable_checkpointing(1000)
-# advanced options:
+# 高级选项：
-# set mode to exactly-once (this is the default)
+# 设置模式为精确一次 (这是默认值)
env.get_checkpoint_config().set_checkpointing_mode(CheckpointingMode.EXACTLY_ONCE)
-# make sure 500 ms of progress happen between checkpoints
+# 确认 checkpoints 之间的时间会进行 500 ms
env.get_checkpoint_config().set_min_pause_between_checkpoints(500)
-# checkpoints have to complete within one minute, or are discarded
+# Checkpoint 必须在一分钟内完成，否则就会被抛弃
env.get_checkpoint_config().set_checkpoint_timeout(60000)
-# allow only one checkpoint to be in progress at the same time
+# 同一时间只允许一个 checkpoint 进行
env.get_checkpoint_config().set_max_concurrent_checkpoints(1)
-# enable externalized checkpoints which are retained after job cancellation
+# 开启在 job 中止后仍然保留的 externalized checkpoints
env.get_checkpoint_config().enable_externalized_checkpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)
-# allow job recovery fallback to checkpoint when there is a more recent savepoint
+# 允许在有更近 savepoint 时回退到 checkpoint
env.get_checkpoint_config().set_prefer_checkpoint_for_recovery(True)
{% endhighlight %}
-### Related Config Options
+### 相关的配置选项
-Some more parameters and/or defaults may be set via `conf/flink-conf.yaml` (see [configuration]({{ site.baseurl }}/ops/config.html) for a full guide):
+更多的属性与默认值能在 `conf/flink-conf.yaml` 中设置（完整教程请阅读 [配置]({{ site.baseurl }}/zh/ops/config.html)）。
{% include generated/checkpointing_configuration.html %}
{% top %}
-## Selecting a State Backend
-
-Flink's [checkpointing mechanism]({{ site.baseurl }}/internals/stream_checkpointing.html) stores consistent snapshots
-of all the state in timers and stateful operators, including connectors, windows, and any [user-defined state](state.html).
-Where the checkpoints are stored (e.g., JobManager memory, file system, database) depends on the configured
-**State Backend**.
+## 选择一个 State Backend
-By default, state is kept in memory in the TaskManagers and checkpoints are stored in memory in the JobManager. For proper persistence of large state,
-Flink supports various approaches for storing and checkpointing state in other state backends. The choice of state backend can be configured via `StreamExecutionEnvironment.setStateBackend(…)`.
+Flink 的 [checkpointing 机制]({{ site.baseurl }}/zh/internals/stream_checkpointing.html) 会将 timer 以及 stateful 的 operator 进行快照，然后存储下来，
+包括连接器（connectors），窗口（windows）以及任何用户[自定义的状态](state.html)。
+Checkpoint 存储在哪里取决于所配置的 **State Backend**（比如 JobManager memory、 file system、 database）。
-See [state backends]({{ site.baseurl }}/ops/state/state_backends.html) for more details on the available state backends and options for job-wide and cluster-wide configuration.
+默认情况下，状态是保持在 TaskManagers 的内存中，checkpoint 保存在 JobManager 的内存中。为了合适地持久化大体量状态，
+Flink 支持各种各样的途径去存储 checkpoint 状态到其他的 state backends 上。通过 `StreamExecutionEnvironment.setStateBackend(…)` 来配置所选的 state backends。
+阅读 [state backends]({{ site.baseurl }}/zh/ops/state/state_backends.html) 来查看在 job 范围和集群范围上可用的 state backends 与选项的更多细节。
-## State Checkpoints in Iterative Jobs
+## 迭代作业中的状态和 checkpoint
-Flink currently only provides processing guarantees for jobs without iterations. Enabling checkpointing on an iterative job causes an exception. In order to force checkpointing on an iterative program the user needs to set a special flag when enabling checkpointing: `env.enableCheckpointing(interval, CheckpointingMode.EXACTLY_ONCE, force = true)`.
+Flink 现在为没有迭代（iterations）的作业提供一致性的处理保证。在迭代作业上开启 checkpoint 会导致异常。为了在迭代程序中强制进行 checkpoint，用户需要在开启 checkpoint 时设置一个特殊的标志： `env.enableCheckpointing(interval, CheckpointingMode.EXACTLY_ONCE, force = true)`。
-Please note that records in flight in the loop edges (and the state changes associated with them) will be lost during failure.
+请注意在环形边上游走的记录（以及与之相关的状态变化）在故障时会丢失。
{% top %}
+## 重启策略
-## Restart Strategies
-
-Flink supports different restart strategies which control how the jobs are restarted in case of a failure. For more
-information, see [Restart Strategies]({{ site.baseurl }}/dev/restart_strategies.html).
+Flink 支持不同的重启策略，来控制 job 万一故障时该如何重启。更多信息请阅读 [重启策略]({{ site.baseurl }}/zh/dev/restart_strategies.html)。
{% top %}
diff --git a/docs/dev/stream/state/index.md b/docs/dev/stream/state/index.md
index 10de348a76d25c6481fedb276bb28bda9f8e9b4c..1d1b436e42a0091250c29e0cbfea0e6ee4516ce7 100644
--- a/docs/dev/stream/state/index.md
+++ b/docs/dev/stream/state/index.md
@@ -25,23 +25,10 @@ specific language governing permissions and limitations
under the License.
-->
-Stateful functions and operators store data across the processing of individual elements/events, making state a critical building block for
-any type of more elaborate operation.
-
-For example:
-
- - When an application searches for certain event patterns, the state will store the sequence of events encountered so far.
- - When aggregating events per minute/hour/day, the state holds the pending aggregates.
- - When training a machine learning model over a stream of data points, the state holds the current version of the model parameters.
- - When historic data needs to be managed, the state allows efficient access to events that occurred in the past.
-
-Flink needs to be aware of the state in order to make state fault tolerant using [checkpoints](checkpointing.html) and to allow [savepoints]({{ site.baseurl }}/ops/state/savepoints.html) of streaming applications.
-
-Knowledge about the state also allows for rescaling Flink applications, meaning that Flink takes care of redistributing state across parallel instances.
-
-The [queryable state](queryable_state.html) feature of Flink allows you to access state from outside of Flink during runtime.
-
-When working with state, it might also be useful to read about [Flink's state backends]({{ site.baseurl }}/ops/state/state_backends.html). Flink provides different state backends that specify how and where state is stored. State can be located on Java's heap or off-heap. Depending on your state backend, Flink can also *manage* the state for the application, meaning Flink deals with the memory management (possibly spilling to disk if necessary) to allow applications to hold very large state. State backends can be configured without changing your application logic.
+In this section you will learn about the APIs that Flink provides for writing
+stateful programs. Please take a look at [Stateful Stream
+Processing]({{site.baseurl}}{% link concepts/stateful-stream-processing.md %})
+to learn about the concepts behind stateful stream processing.
{% top %}
diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md
index c6d4c95a7dff43b3e0e09382839b91349be3ac26..028ad8fe17669b99306326e21353cd4bfd73e3ab 100644
--- a/docs/dev/stream/state/state.md
+++ b/docs/dev/stream/state/state.md
@@ -22,66 +22,17 @@ specific language governing permissions and limitations
under the License.
-->
-This document explains how to use Flink's state abstractions when developing an application.
+In this section you will learn about the APIs that Flink provides for writing
+stateful programs. Please take a look at [Stateful Stream
+Processing]({{site.baseurl}}{% link concepts/stateful-stream-processing.md %})
+to learn about the concepts behind stateful stream processing.
* ToC
{:toc}
-## Keyed State and Operator State
+## Using Keyed State
-There are two basic kinds of state in Flink: `Keyed State` and `Operator State`.
-
-### Keyed State
-
-*Keyed State* is always relative to keys and can only be used in functions and operators on a `KeyedStream`.
-
-You can think of Keyed State as Operator State that has been partitioned,
-or sharded, with exactly one state-partition per key.
-Each keyed-state is logically bound to a unique
-composite of , and since each key
-"belongs" to exactly one parallel instance of a keyed operator, we can
-think of this simply as .
-
-Keyed State is further organized into so-called *Key Groups*. Key Groups are the
-atomic unit by which Flink can redistribute Keyed State;
-there are exactly as many Key Groups as the defined maximum parallelism.
-During execution each parallel instance of a keyed operator works with the keys
-for one or more Key Groups.
-
-### Operator State
-
-With *Operator State* (or *non-keyed state*), each operator state is
-bound to one parallel operator instance.
-The [Kafka Connector]({{ site.baseurl }}/dev/connectors/kafka.html) is a good motivating example for the use of Operator State
-in Flink. Each parallel instance of the Kafka consumer maintains a map
-of topic partitions and offsets as its Operator State.
-
-The Operator State interfaces support redistributing state among
-parallel operator instances when the parallelism is changed. There can be different schemes for doing this redistribution.
-
-## Raw and Managed State
-
-*Keyed State* and *Operator State* exist in two forms: *managed* and *raw*.
-
-*Managed State* is represented in data structures controlled by the Flink runtime, such as internal hash tables, or RocksDB.
-Examples are "ValueState", "ListState", etc. Flink's runtime encodes
-the states and writes them into the checkpoints.
-
-*Raw State* is state that operators keep in their own data structures. When checkpointed, they only write a sequence of bytes into
-the checkpoint. Flink knows nothing about the state's data structures and sees only the raw bytes.
-
-All datastream functions can use managed state, but the raw state interfaces can only be used when implementing operators.
-Using managed state (rather than raw state) is recommended, since with
-managed state Flink is able to automatically redistribute state when the parallelism is
-changed, and also do better memory management.
-
-Attention If your managed state needs custom serialization logic, please see
-the [corresponding guide](custom_serialization.html) in order to ensure future compatibility. Flink's default serializers
-don't need special treatment.
-
-## Using Managed Keyed State
-
-The managed keyed state interface provides access to different types of state that are all scoped to
+The keyed state interfaces provides access to different types of state that are all scoped to
the key of the current input element. This means that this type of state can only be used
on a `KeyedStream`, which can be created via `stream.keyBy(…)`.
@@ -115,6 +66,7 @@ added using `add(T)` are folded into an aggregate using a specified `FoldFunctio
retrieve an `Iterable` over all currently stored mappings. Mappings are added using `put(UK, UV)` or
`putAll(Map)`. The value associated with a user key can be retrieved using `get(UK)`. The iterable
views for mappings, keys and values can be retrieved using `entries()`, `keys()` and `values()` respectively.
+You can also use `isEmpty()` to check whether this map contains any key-value mappings.
All types of state also have a method `clear()` that clears the state for the currently
active key, i.e. the key of the input element.
@@ -254,7 +206,7 @@ object ExampleCountWindowAverage extends App {
.print()
// the printed output will be (1,4) and (1,5)
- env.execute("ExampleManagedState")
+ env.execute("ExampleKeyedState")
}
{% endhighlight %}
@@ -355,79 +307,71 @@ If the serializer does not support null values, it can be wrapped with `Nullable
#### Cleanup of Expired State
-By default, expired values are only removed when they are read out explicitly,
-e.g. by calling `ValueState.value()`.
-
-Attention This means that by default if expired state is not read,
-it won't be removed, possibly leading to ever growing state. This might change in future releases.
-
-##### Cleanup in full snapshot
-
-Additionally, you can activate the cleanup at the moment of taking the full state snapshot which
-will reduce its size. The local state is not cleaned up under the current implementation
-but it will not include the removed expired state in case of restoration from the previous snapshot.
-It can be configured in `StateTtlConfig`:
+By default, expired values are explicitly removed on read, such as `ValueState#value`, and periodically garbage collected
+in the background if supported by the configured state backend. Background cleanup can be disabled in the `StateTtlConfig`:
-This option is not applicable for the incremental checkpointing in the RocksDB state backend.
-
-**Notes:**
-- For existing jobs, this cleanup strategy can be activated or deactivated anytime in `StateTtlConfig`,
-e.g. after restart from savepoint.
+For more fine-grained control over some special cleanup in background, you can configure it separately as described below.
+Currently, heap state backend relies on incremental cleanup and RocksDB backend uses compaction filter for background cleanup.
-#### Cleanup in background
+##### Cleanup in full snapshot
-Besides cleanup in full snapshot, you can also activate the cleanup in background. The following option
-will activate a default background cleanup in StateTtlConfig if it is supported for the used backend:
+Additionally, you can activate the cleanup at the moment of taking the full state snapshot which
+will reduce its size. The local state is not cleaned up under the current implementation
+but it will not include the removed expired state in case of restoration from the previous snapshot.
+It can be configured in `StateTtlConfig`:
-For more fine-grained control over some special cleanup in background, you can configure it separately as described below.
-Currently, heap state backend relies on incremental cleanup and RocksDB backend uses compaction filter for background cleanup.
+This option is not applicable for the incremental checkpointing in the RocksDB state backend.
+
+**Notes:**
+- For existing jobs, this cleanup strategy can be activated or deactivated anytime in `StateTtlConfig`,
+e.g. after restart from savepoint.
##### Incremental cleanup
@@ -438,7 +382,7 @@ The storage backend keeps a lazy global iterator for this state over all its ent
Every time incremental cleanup is triggered, the iterator is advanced.
The traversed state entries are checked and expired ones are cleaned up.
-This feature can be activated in `StateTtlConfig`:
+This feature can be configured in `StateTtlConfig`:
@@ -462,9 +406,9 @@ val ttlConfig = StateTtlConfig
This strategy has two parameters. The first one is number of checked state entries per each cleanup triggering.
-If enabled, it is always triggered per each state access.
+It is always triggered per each state access.
The second parameter defines whether to trigger cleanup additionally per each record processing.
-If you enable the default background cleanup then this strategy will be activated for heap backend with 5 checked entries and without cleanup per record processing.
+The default background cleanup for heap backend checks 5 entries without cleanup per record processing.
**Notes:**
- If no access happens to the state or no records are processed, expired state will persist.
@@ -478,15 +422,12 @@ e.g. after restart from savepoint.
##### Cleanup during RocksDB compaction
-If RocksDB state backend is used, another cleanup strategy is to activate Flink specific compaction filter.
+If the RocksDB state backend is used, a Flink specific compaction filter will be called for the background cleanup.
RocksDB periodically runs asynchronous compactions to merge state updates and reduce storage.
Flink compaction filter checks expiration timestamp of state entries with TTL
and excludes expired values.
-This feature is disabled by default. It has to be firstly activated for the RocksDB backend
-by setting Flink configuration option `state.backend.rocksdb.ttl.compaction.filter.enabled`
-or by calling `RocksDBStateBackend::enableTtlCompactionFilter` if a custom RocksDB state backend is created for a job.
-Then any state with TTL can be configured to use the filter:
+This feature can be configured in `StateTtlConfig`:
@@ -518,7 +459,7 @@ You can change it and pass a custom value to
`StateTtlConfig.newBuilder(...).cleanupInRocksdbCompactFilter(long queryTimeAfterNumEntries)` method.
Updating the timestamp more often can improve cleanup speed
but it decreases compaction performance because it uses JNI call from native code.
-If you enable the default background cleanup then this strategy will be activated for RocksDB backend and the current timestamp will be queried each time 1000 entries have been processed.
+The default background cleanup for RocksDB backend queries the current timestamp each time 1000 entries have been processed.
You can activate debug logs from the native code of RocksDB filter
by activating debug level for `FlinkCompactionFilter`:
@@ -555,11 +496,47 @@ val counts: DataStream[(String, Int)] = stream
})
{% endhighlight %}
-## Using Managed Operator State
+## Operator State
+
+*Operator State* (or *non-keyed state*) is state that is is bound to one
+parallel operator instance. The [Kafka Connector]({{ site.baseurl }}{% link
+dev/connectors/kafka.md %}) is a good motivating example for the use of
+Operator State in Flink. Each parallel instance of the Kafka consumer maintains
+a map of topic partitions and offsets as its Operator State.
+
+The Operator State interfaces support redistributing state among parallel
+operator instances when the parallelism is changed. There are different schemes
+for doing this redistribution.
-To use managed operator state, a stateful function can implement either the more general `CheckpointedFunction`
+In a typical stateful Flink Application you don't need operators state. It is
+mostly a special type of state that is used in source/sink implementations and
+scenarios where you don't have a key by which state can be partitioned.
+
+## Broadcast State
+
+*Broadcast State* is a special type of *Operator State*. It was introduced to
+support use cases where records of one stream need to be broadcasted to all
+downstream tasks, where they are used to maintain the same state among all
+subtasks. This state can then be accessed while processing records of a second
+stream. As an example where broadcast state can emerge as a natural fit, one
+can imagine a low-throughput stream containing a set of rules which we want to
+evaluate against all elements coming from another stream. Having the above type
+of use cases in mind, broadcast state differs from the rest of operator states
+in that:
+
+ 1. it has a map format,
+ 2. it is only available to specific operators that have as inputs a
+ *broadcasted* stream and a *non-broadcasted* one, and
+ 3. such an operator can have *multiple broadcast states* with different names.
+
+{% top %}
+
+## Using Operator State
+
+To use operator state, a stateful function can implement either the more general `CheckpointedFunction`
interface, or the `ListCheckpointed` interface.
+
#### CheckpointedFunction
The `CheckpointedFunction` interface provides access to non-keyed state with different
@@ -576,7 +553,7 @@ is called every time the user-defined function is initialized, be that when the
or be that when the function is actually recovering from an earlier checkpoint. Given this, `initializeState()` is not
only the place where different types of state are initialized, but also where state recovery logic is included.
-Currently, list-style managed operator state is supported. The state
+Currently, list-style operator state is supported. The state
is expected to be a `List` of *serializable* objects, independent from each other,
thus eligible for redistribution upon rescaling. In other words, these objects are the finest granularity at which
non-keyed state can be redistributed. Depending on the state accessing method,
diff --git a/docs/dev/stream/state/state.zh.md b/docs/dev/stream/state/state.zh.md
index 78da6410d66a258be3cf1f1df0138e10941f1434..dad89a35fcd6ce01fd7f2e2a1126ce8087fe990a 100644
--- a/docs/dev/stream/state/state.zh.md
+++ b/docs/dev/stream/state/state.zh.md
@@ -87,7 +87,7 @@ managed keyed state 接口提供不同类型状态的访问接口，这些状态
接口与 `ListState` 类似，但使用`add（T）`添加的元素会用指定的 `FoldFunction` 折叠成聚合值。
* `MapState`: 维护了一个映射列表。 你可以添加键值对到状态中，也可以获得反映当前所有映射的迭代器。使用 `put(UK，UV)` 或者 `putAll(Map)` 添加映射。
- 使用 `get(UK)` 检索特定 key。 使用 `entries()`，`keys()` 和 `values()` 分别检索映射、键和值的可迭代视图。
+ 使用 `get(UK)` 检索特定 key。 使用 `entries()`，`keys()` 和 `values()` 分别检索映射、键和值的可迭代视图。你还可以通过 `isEmpty()` 来判断是否包含任何键值对。
所有类型的状态还有一个`clear()` 方法，清除当前 key 下的状态数据，也就是当前输入元素的 key。
@@ -304,23 +304,17 @@ Heap state backend 会额外存储一个包括用户状态以及时间戳的 Jav
#### 过期数据的清理
-默认情况下，仅在用户读取过期数据时才会删除过对应的状态，例如调用 `ValueState.value()`。
+默认情况下，过期数据会在读取的时候被删除，例如 `ValueState#value`，同时会有后台线程定期清理（如果 StateBackend 支持的话）。可以通过 `StateTtlConfig` 配置关闭后台清理：
-注意 默认情况下，如果不显示读取过期数据，则不会进行删除，可能导致状态持续增加。这种行为在未来可能会进行改变。
-
-##### 全量快照时清理
-
-此外，你可以在进行全量快照时进行清理，这将减少快照的大小。 当前实现中不会清除本地状态，但是在从先前快照恢复时则会移除过期数据。可以通过 `StateTtlConfig` 配置：
-这个配置对 RocksDB 增量 checkpoint 无效。
-
-**注意:**
-- 这种清理方式可以在任何时候通过 `StateTtlConfig` 启用或者关闭，比如在从 savepoint 恢复时。
+可以按照如下所示配置更细粒度的后台清理策略。当前的实现中 `HeapStateBackend` 依赖增量数据清理，`RocksDBStateBackend` 利用压缩过滤器进行后台清理。
-#### Cleanup in background
+#### 全量快照时进行清理
-Besides cleanup in full snapshot, you can also activate the cleanup in background. The following option
-will activate a default background cleanup in StateTtlConfig if it is supported for the used backend:
+另外，你可以启用全量快照时进行清理的策略，这可以减少整个快照的大小。当前实现中不会清理本地的状态，但从上次快照恢复时，不会恢复那些已经删除的过期数据。
+该策略可以通过 `StateTtlConfig` 配置进行配置：
-For more fine-grained control over some special cleanup in background, you can configure it separately as described below.
-Currently, heap state backend relies on incremental cleanup and RocksDB backend uses compaction filter for background cleanup.
+这种策略在 `RocksDBStateBackend` 的增量 checkpoint 模式下无效。
+
+**注意:**
+- 这种清理方式可以在任何时候通过 `StateTtlConfig` 启用或者关闭，比如在从 savepoint 恢复时。
##### 增量数据清理
另外可以选择增量式清理状态数据，在状态访问或/和处理时进行。如果某个状态开启了该清理策略，则会在存储后端保留一个所有状态的惰性全局迭代器。
每次触发增量清理时，从迭代器中选择已经过期的数进行清理。
-该特性可以通过 `StateTtlConfig` 进行启用：
+该特性可以通过 `StateTtlConfig` 进行配置：
@@ -400,8 +397,8 @@ val ttlConfig = StateTtlConfig
-该策略有两个参数。 第一个是每次清理时检查状态的条目数。如果启用，则始终按每个状态访问触发。第二个参数表示是否在处理每条记录时触发清理。
-If you enable the default background cleanup then this strategy will be activated for heap backend with 5 checked entries and without cleanup per record processing.
+该策略有两个参数。 第一个是每次清理时检查状态的条目数，在每个状态访问时触发。第二个参数表示是否在处理每条记录时触发清理。
+Heap backend 默认会检查 5 条状态，并且关闭在每条记录时触发清理。
**注意:**
- 如果没有 state 访问，也没有处理数据，则不会清理过期数据。
@@ -412,11 +409,10 @@ If you enable the default background cleanup then this strategy will be activate
##### 在 RocksDB 压缩时清理
-如果使用 RocksDB state backend，还支持 Flink 为 RocksDB 定制的压缩过滤器。RocksDB 会周期性的对数据进行合并压缩从而减少存储空间。
-Flink 压缩过滤器会在压缩时过滤掉已经过期的状态数据。
+如果使用 RocksDB state backend，则会启用 Flink 为 RocksDB 定制的压缩过滤器。RocksDB 会周期性的对数据进行合并压缩从而减少存储空间。
+Flink 提供的 RocksDB 压缩过滤器会在压缩时过滤掉已经过期的状态数据。
-该特性默认是关闭的，可以通过 Flink 的配置项 `state.backend.rocksdb.ttl.compaction.filter.enabled` 或者调用 `RocksDBStateBackend::enableTtlCompactionFilter`
-启用该特性。然后通过如下方式让任何具有 TTL 配置的状态使用过滤器：
+该特性可以通过 `StateTtlConfig` 进行配置：
@@ -442,13 +438,10 @@ val ttlConfig = StateTtlConfig
-RocksDB compaction filter will query current timestamp, used to check expiration, from Flink every time
-after processing certain number of state entries.
-You can change it and pass a custom value to
-`StateTtlConfig.newBuilder(...).cleanupInRocksdbCompactFilter(long queryTimeAfterNumEntries)` method.
-Updating the timestamp more often can improve cleanup speed
-but it decreases compaction performance because it uses JNI call from native code.
-If you enable the default background cleanup then this strategy will be activated for RocksDB backend and the current timestamp will be queried each time 1000 entries have been processed.
+Flink 处理一定条数的状态数据后，会使用当前时间戳来检测 RocksDB 中的状态是否已经过期，
+你可以通过 `StateTtlConfig.newBuilder(...).cleanupInRocksdbCompactFilter(long queryTimeAfterNumEntries)` 方法指定处理状态的条数。
+时间戳更新的越频繁，状态的清理越及时，但由于压缩会有调用 JNI 的开销，因此会影响整体的压缩性能。
+RocksDB backend 的默认后台清理策略会每处理 1000 条数据进行一次。
你还可以通过配置开启 RocksDB 过滤器的 debug 日志：
`log4j.logger.org.rocksdb.FlinkCompactionFilter=DEBUG`
diff --git a/docs/dev/stream/testing.md b/docs/dev/stream/testing.md
index 2bd0096bbf77b8191fcdf3de0b6bb432cd13e945..8ed77627f7dfd9e80a8c3d2378976952f2bdda60 100644
--- a/docs/dev/stream/testing.md
+++ b/docs/dev/stream/testing.md
@@ -147,7 +147,7 @@ class IncrementFlatMapFunctionTest extends FlatSpec with MockFactory {
Testing the functionality of a user-defined function, which makes use of managed state or timers is more difficult because it involves testing the interaction between the user code and Flink's runtime.
For this Flink comes with a collection of so called test harnesses, which can be used to test such user-defined functions as well as custom operators:
-* `OneInputStreamOperatorTestHarness` (for operators on `DataStreams`s)
+* `OneInputStreamOperatorTestHarness` (for operators on `DataStream`s)
* `KeyedOneInputStreamOperatorTestHarness` (for operators on `KeyedStream`s)
* `TwoInputStreamOperatorTestHarness` (for operators of `ConnectedStreams` of two `DataStream`s)
* `KeyedTwoInputStreamOperatorTestHarness` (for operators on `ConnectedStreams` of two `KeyedStream`s)
@@ -332,6 +332,90 @@ Many more examples for the usage of these test harnesses can be found in the Fli
Note Be aware that `AbstractStreamOperatorTestHarness` and its derived classes are currently not part of the public API and can be subject to change.
+#### Unit Testing ProcessFunction
+
+Given its importance, in addition to the previous test harnesses that can be used directly to test a `ProcessFunction`, Flink provides a test harness factory named `ProcessFunctionTestHarnesses` that allows for easier test harness instantiation. Considering this example:
+
+Note Be aware that to use this test harness, you also need to introduce the dependencies mentioned in the last section.
+
+
+
+It is very easy to unit test such a function with `ProcessFunctionTestHarnesses` by passing suitable arguments and verifying the output.
+
+
+
+{% highlight java %}
+public class PassThroughProcessFunctionTest {
+
+ @Test
+ public void testPassThrough() throws Exception {
+
+ //instantiate user-defined function
+ PassThroughProcessFunction processFunction = new PassThroughProcessFunction();
+
+ // wrap user defined function into a the corresponding operator
+ OneInputStreamOperatorTestHarness harness = ProcessFunctionTestHarnesses
+ .forProcessFunction(processFunction);
+
+ //push (timestamped) elements into the operator (and hence user defined function)
+ harness.processElement(1, 10);
+
+ //retrieve list of emitted records for assertions
+ assertEquals(harness.extractOutputValues(), Collections.singletonList(1));
+ }
+}
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+class PassThroughProcessFunctionTest extends FlatSpec with Matchers {
+
+ "PassThroughProcessFunction" should "forward values" in {
+
+ //instantiate user-defined function
+ val processFunction = new PassThroughProcessFunction
+
+ // wrap user defined function into a the corresponding operator
+ val harness = ProcessFunctionTestHarnesses.forProcessFunction(processFunction)
+
+ //push (timestamped) elements into the operator (and hence user defined function)
+ harness.processElement(1, 10)
+
+ //retrieve list of emitted records for assertions
+ harness.extractOutputValues() should contain (1)
+ }
+}
+{% endhighlight %}
+
+
+
+For more examples on how to use the `ProcessFunctionTestHarnesses` in order to test the different flavours of the `ProcessFunction`, e.g. `KeyedProcessFunction`, `KeyedCoProcessFunction`, `BroadcastProcessFunction`, etc, the user is encouraged to look at the `ProcessFunctionTestHarnessesTest`.
+
## Testing Flink Jobs
### JUnit Rule `MiniClusterWithClientResource`
@@ -494,7 +578,7 @@ A few remarks on integration testing with `MiniClusterWithClientResource`:
Communicating with operators instantiated by a local Flink mini cluster via static variables is one way around this issue.
Alternatively, you could write the data to files in a temporary directory with your test sink.
-* You can implement a custom *parallel* source function for emitting watermarks if your job uses event timer timers.
+* You can implement a custom *parallel* source function for emitting watermarks if your job uses event time timers.
* It is recommended to always test your pipelines locally with a parallelism > 1 to identify bugs which only surface for the pipelines executed in parallel.
diff --git a/docs/dev/stream/testing.zh.md b/docs/dev/stream/testing.zh.md
index 2264ec40cdd6d067d092a2f7cbe4d2f6f5b601fe..2b019920a0d5262d58b2e09a2e08ee458908f3a2 100644
--- a/docs/dev/stream/testing.zh.md
+++ b/docs/dev/stream/testing.zh.md
@@ -147,7 +147,7 @@ class IncrementFlatMapFunctionTest extends FlatSpec with MockFactory {
Testing the functionality of a user-defined function, which makes use of managed state or timers is more difficult because it involves testing the interaction between the user code and Flink's runtime.
For this Flink comes with a collection of so called test harnesses, which can be used to test such user-defined functions as well as custom operators:
-* `OneInputStreamOperatorTestHarness` (for operators on `DataStreams`s)
+* `OneInputStreamOperatorTestHarness` (for operators on `DataStream`s)
* `KeyedOneInputStreamOperatorTestHarness` (for operators on `KeyedStream`s)
* `TwoInputStreamOperatorTestHarness` (for operators of `ConnectedStreams` of two `DataStream`s)
* `KeyedTwoInputStreamOperatorTestHarness` (for operators on `ConnectedStreams` of two `KeyedStream`s)
@@ -332,6 +332,90 @@ Many more examples for the usage of these test harnesses can be found in the Fli
Note Be aware that `AbstractStreamOperatorTestHarness` and its derived classes are currently not part of the public API and can be subject to change.
+#### Unit Testing ProcessFunction
+
+Given its importance, in addition to the previous test harnesses that can be used directly to test a `ProcessFunction`, Flink provides a test harness factory named `ProcessFunctionTestHarnesses` that allows for easier test harness instantiation. Considering this example:
+
+Note Be aware that to use this test harness, you also need to introduce the dependencies mentioned in the last section.
+
+
+
+It is very easy to unit test such a function with `ProcessFunctionTestHarnesses` by passing suitable arguments and verifying the output.
+
+
+
+{% highlight java %}
+public class PassThroughProcessFunctionTest {
+
+ @Test
+ public void testPassThrough() throws Exception {
+
+ //instantiate user-defined function
+ PassThroughProcessFunction processFunction = new PassThroughProcessFunction();
+
+ // wrap user defined function into a the corresponding operator
+ OneInputStreamOperatorTestHarness harness = ProcessFunctionTestHarnesses
+ .forProcessFunction(processFunction);
+
+ //push (timestamped) elements into the operator (and hence user defined function)
+ harness.processElement(1, 10);
+
+ //retrieve list of emitted records for assertions
+ assertEquals(harness.extractOutputValues(), Collections.singletonList(1));
+ }
+}
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+class PassThroughProcessFunctionTest extends FlatSpec with Matchers {
+
+ "PassThroughProcessFunction" should "forward values" in {
+
+ //instantiate user-defined function
+ val processFunction = new PassThroughProcessFunction
+
+ // wrap user defined function into a the corresponding operator
+ val harness = ProcessFunctionTestHarnesses.forProcessFunction(processFunction)
+
+ //push (timestamped) elements into the operator (and hence user defined function)
+ harness.processElement(1, 10)
+
+ //retrieve list of emitted records for assertions
+ harness.extractOutputValues() should contain (1)
+ }
+}
+{% endhighlight %}
+
+
+
+For more examples on how to use the `ProcessFunctionTestHarnesses` in order to test the different flavours of the `ProcessFunction`, e.g. `KeyedProcessFunction`, `KeyedCoProcessFunction`, `BroadcastProcessFunction`, etc, the user is encouraged to look at the `ProcessFunctionTestHarnessesTest`.
+
## Testing Flink Jobs
### JUnit Rule `MiniClusterWithClientResource`
@@ -494,7 +578,7 @@ A few remarks on integration testing with `MiniClusterWithClientResource`:
Communicating with operators instantiated by a local Flink mini cluster via static variables is one way around this issue.
Alternatively, you could write the data to files in a temporary directory with your test sink.
-* You can implement a custom *parallel* source function for emitting watermarks if your job uses event timer timers.
+* You can implement a custom *parallel* source function for emitting watermarks if your job uses event time timers.
* It is recommended to always test your pipelines locally with a parallelism > 1 to identify bugs which only surface for the pipelines executed in parallel.
diff --git a/docs/dev/table/catalogs.md b/docs/dev/table/catalogs.md
index 52503e6af7e025bba3157ac92475bc870a9e7a2b..862ae3a9e1323b3428fefc2e80f5c9d419ad4dc3 100644
--- a/docs/dev/table/catalogs.md
+++ b/docs/dev/table/catalogs.md
@@ -1,8 +1,7 @@
---
title: "Catalogs"
-is_beta: true
nav-parent_id: tableapi
-nav-pos: 100
+nav-pos: 80
---
+
+Flink Table API & SQL empowers users to do data transformations with functions.
+
+* This will be replaced by the TOC
+{:toc}
+
+Types of Functions
+------------------
+
+There are two dimensions to classify functions in Flink.
+
+One dimension is system (or built-in) functions v.s. catalog functions. System functions have no namespace and can be
+referenced with just their names. Catalog functions belong to a catalog and database therefore they have catalog and database
+namespaces, they can be referenced by either fully/partially qualified name (`catalog.db.func` or `db.func`) or just the
+function name.
+
+The other dimension is temporary functions v.s. persistent functions. Temporary functions are volatile and only live up to
+ lifespan of a session, they are always created by users. Persistent functions live across lifespan of sessions, they are either
+ provided by the system or persisted in catalogs.
+
+The two dimensions give Flink users 4 categories of functions:
+
+1. Temporary system functions
+2. System functions
+3. Temporary catalog functions
+4. Catalog functions
+
+Referencing Functions
+---------------------
+
+There are two ways users can reference a function in Flink - referencing function precisely or ambiguously.
+
+## Precise Function Reference
+
+Precise function reference empowers users to use catalog functions specifically, and across catalog and across database,
+e.g. `select mycatalog.mydb.myfunc(x) from mytable` and `select mydb.myfunc(x) from mytable`.
+
+This is only supported starting from Flink 1.10.
+
+## Ambiguous Function Reference
+
+In ambiguous function reference, users just specify the function's name in SQL query, e.g. `select myfunc(x) from mytable`.
+
+
+Function Resolution Order
+-------------------------
+
+The resolution order only matters when there are functions of different types but the same name,
+e.g. when there’re three functions all named “myfunc” but are of temporary catalog, catalog, and system function respectively.
+If there’s no function name collision, functions will just be resolved to the sole one.
+
+## Precise Function Reference
+
+Because system functions don’t have namespaces, a precise function reference in Flink must be pointing to either a temporary catalog
+function or a catalog function.
+
+The resolution order is:
+
+1. Temporary catalog function
+2. Catalog function
+
+## Ambiguous Function Reference
+
+The resolution order is:
+
+1. Temporary system function
+2. System function
+3. Temporary catalog function, in the current catalog and current database of the session
+4. Catalog function, in the current catalog and current database of the session
diff --git a/docs/dev/table/functions/index.zh.md b/docs/dev/table/functions/index.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..ac3d9923f2d19a4c4f71ca5ae9a9fe5db7d1d99b
--- /dev/null
+++ b/docs/dev/table/functions/index.zh.md
@@ -0,0 +1,97 @@
+---
+title: "Functions"
+nav-id: table_functions
+nav-parent_id: tableapi
+nav-pos: 60
+nav-show_overview: true
+---
+
+
+Flink Table API & SQL empowers users to do data transformations with functions.
+
+* This will be replaced by the TOC
+{:toc}
+
+Types of Functions
+------------------
+
+There are two dimensions to classify functions in Flink.
+
+One dimension is system (or built-in) functions v.s. catalog functions. System functions have no namespace and can be
+referenced with just their names. Catalog functions belong to a catalog and database therefore they have catalog and database
+namespaces, they can be referenced by either fully/partially qualified name (`catalog.db.func` or `db.func`) or just the
+function name.
+
+The other dimension is temporary functions v.s. persistent functions. Temporary functions are volatile and only live up to
+ lifespan of a session, they are always created by users. Persistent functions live across lifespan of sessions, they are either
+ provided by the system or persisted in catalogs.
+
+The two dimensions give Flink users 4 categories of functions:
+
+1. Temporary system functions
+2. System functions
+3. Temporary catalog functions
+4. Catalog functions
+
+Note that system functions always precede catalog's, and temporary functions always precede persistent on their own dimension
+in function resolution order explained below.
+
+Referencing Functions
+---------------------
+
+There are two ways users can reference a function in Flink - referencing function precisely or ambiguously.
+
+## Precise Function Reference
+
+Precise function reference empowers users to use catalog functions specifically, and across catalog and across database,
+e.g. `select mycatalog.mydb.myfunc(x) from mytable` and `select mydb.myfunc(x) from mytable`.
+
+This is only supported starting from Flink 1.10.
+
+## Ambiguous Function Reference
+
+In ambiguous function reference, users just specify the function's name in SQL query, e.g. `select myfunc(x) from mytable`.
+
+
+Function Resolution Order
+-------------------------
+
+The resolution order only matters when there are functions of different types but the same name,
+e.g. when there’re three functions all named “myfunc” but are of temporary catalog, catalog, and system function respectively.
+If there’s no function name collision, functions will just be resolved to the sole one.
+
+## Precise Function Reference
+
+Because system functions don’t have namespaces, a precise function reference in Flink must be pointing to either a temporary catalog
+function or a catalog function.
+
+The resolution order is:
+
+1. Temporary catalog function
+2. Catalog function
+
+## Ambiguous Function Reference
+
+The resolution order is:
+
+1. Temporary system function
+2. System function
+3. Temporary catalog function, in the current catalog and current database of the session
+4. Catalog function, in the current catalog and current database of the session
diff --git a/docs/dev/table/functions.md b/docs/dev/table/functions/systemFunctions.md
similarity index 91%
rename from docs/dev/table/functions.md
rename to docs/dev/table/functions/systemFunctions.md
index c2fcaa5706022e34e1ad04334b012a22d6626199..b78f5e01e8594e1994aebc6b7992e2e37ae01a9b 100644
--- a/docs/dev/table/functions.md
+++ b/docs/dev/table/functions/systemFunctions.md
@@ -1,6 +1,6 @@
---
-title: "Built-In Functions"
-nav-parent_id: tableapi
+title: "System (Built-in) Functions"
+nav-parent_id: table_functions
nav-pos: 31
---
+
+Hive Metastore has evolved into the de facto metadata hub over the years in Hadoop ecosystem. Many companies have a single
+Hive Metastore service instance in their production to manage all of their metadata, either Hive metadata or non-Hive metadata,
+ as the source of truth.
+
+For users who have both Hive and Flink deployments, `HiveCatalog` enables them to use Hive Metastore to manage Flink's metadata.
+
+For users who have just Flink deployment, `HiveCatalog` is the only persistent catalog provided out-of-box by Flink.
+Without a persistent catalog, users using [Flink SQL CREATE DDL]({{ site.baseurl }}/dev/table/sql/create.html) have to repeatedly
+create meta-objects like a Kafka table in each session, which wastes a lot of time. `HiveCatalog` fills this gap by empowering
+users to create tables and other meta-objects only once, and reference and manage them with convenience later on across sessions.
+
+
+## Set up HiveCatalog
+
+### Dependencies
+
+Setting up a `HiveCatalog` in Flink requires the same [dependencies]({{ site.baseurl }}/dev/table/hive/#dependencies)
+as those of an overall Flink-Hive integration.
+
+### Configuration
+
+Setting up a `HiveCatalog` in Flink requires the same [configuration]({{ site.baseurl }}/dev/table/hive/#connecting-to-hive)
+as those of an overall Flink-Hive integration.
+
+
+## How to use HiveCatalog
+
+Once configured properly, `HiveCatalog` should just work out of box. Users can create Flink meta-objects with DDL, and shoud
+see them immediately afterwards.
+
+`HiveCatalog` can be used to handle two kinds of tables: Hive-compatible tables and generic tables. Hive-compatible tables
+are those stored in a Hive-compatible way, in terms of both metadata and data in the storage layer. Therefore, Hive-compatible tables
+created via Flink can be queried from Hive side.
+
+Generic tables, on the other hand, are specific to Flink. When creating generic tables with `HiveCatalog`, we're just using
+HMS to persist the metadata. While these tables are visible to Hive, it's unlikely Hive is able to understand
+the metadata. And therefore using such tables in Hive leads to undefined behavior.
+
+Flink uses the property '*is_generic*' to tell whether a table is Hive-compatible or generic. When creating a table with
+`HiveCatalog`, it's by default considered generic. If you'd like to create a Hive-compatible table, make sure to set
+`is_generic` to false in your table properties.
+
+As stated above, generic tables shouldn't be used from Hive. In Hive CLI, you can call `DESCRIBE FORMATTED` for a table and
+decide whether it's generic or not by checking the `is_generic` property. Generic tables will have `is_generic=true`.
+
+### Example
+
+We will walk through a simple example here.
+
+#### step 1: set up a Hive Metastore
+
+Have a Hive Metastore running.
+
+Here, we set up a local Hive Metastore and our `hive-site.xml` file in local path `/opt/hive-conf/hive-site.xml`.
+We have some configs like the following:
+
+{% highlight xml %}
+
+
+
+ javax.jdo.option.ConnectionURL
+ jdbc:mysql://localhost/metastore?createDatabaseIfNotExist=true
+ metadata is stored in a MySQL server
+
+
+
+ javax.jdo.option.ConnectionDriverName
+ com.mysql.jdbc.Driver
+ MySQL JDBC driver class
+
+
+
+ javax.jdo.option.ConnectionUserName
+ ...
+ user name for connecting to mysql server
+
+
+
+ javax.jdo.option.ConnectionPassword
+ ...
+ password for connecting to mysql server
+
+
+
+ hive.metastore.uris
+ thrift://localhost:9083
+ IP address (or fully-qualified domain name) and port of the metastore host
+
+
+
+ hive.metastore.schema.verification
+ true
+
+
+
+{% endhighlight %}
+
+
+Test connection to the HMS with Hive Cli. Running some commands, we can see we have a database named `default` and there's no table in it.
+
+
+{% highlight bash %}
+
+hive> show databases;
+OK
+default
+Time taken: 0.032 seconds, Fetched: 1 row(s)
+
+hive> show tables;
+OK
+Time taken: 0.028 seconds, Fetched: 0 row(s)
+{% endhighlight %}
+
+
+#### step 2: configure Flink cluster and SQL CLI
+
+Add all Hive dependencies to `/lib` dir in Flink distribution, and modify SQL CLI's yaml config file `sql-cli-defaults.yaml` as following:
+
+{% highlight yaml %}
+
+execution:
+ planner: blink
+ type: streaming
+ ...
+ current-catalog: myhive # set the HiveCatalog as the current catalog of the session
+ current-database: mydatabase
+
+catalogs:
+ - name: myhive
+ type: hive
+ hive-conf-dir: /opt/hive-conf # contains hive-site.xml
+ hive-version: 2.3.4
+{% endhighlight %}
+
+
+#### step 3: set up a Kafka cluster
+
+Bootstrap a local Kafka 2.3.0 cluster with a topic named "test", and produce some simple data to the topic as tuple of name and age.
+
+{% highlight bash %}
+
+localhost$ bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
+>tom,15
+>john,21
+
+{% endhighlight %}
+
+
+These message can be seen by starting a Kafka console consumer.
+
+{% highlight bash %}
+localhost$ bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
+
+tom,15
+john,21
+
+{% endhighlight %}
+
+
+#### step 4: start SQL Client, and create a Kafka table with Flink SQL DDL
+
+Start Flink SQL Client, create a simple Kafka 2.3.0 table via DDL, and verify its schema.
+
+{% highlight bash %}
+
+Flink SQL> CREATE TABLE mykafka (name String, age Int) WITH (
+ 'connector.type' = 'kafka',
+ 'connector.version' = 'universal',
+ 'connector.topic' = 'test',
+ 'connector.properties.zookeeper.connect' = 'localhost:2181',
+ 'connector.properties.bootstrap.servers' = 'localhost:9092',
+ 'format.type' = 'csv',
+ 'update-mode' = 'append'
+);
+[INFO] Table has been created.
+
+Flink SQL> DESCRIBE mykafka;
+root
+ |-- name: STRING
+ |-- age: INT
+
+{% endhighlight %}
+
+Verify the table is also visible to Hive via Hive Cli, and note that the table has property `is_generic=true`:
+
+{% highlight bash %}
+hive> show tables;
+OK
+mykafka
+Time taken: 0.038 seconds, Fetched: 1 row(s)
+
+hive> describe formatted mykafka;
+OK
+# col_name data_type comment
+
+
+# Detailed Table Information
+Database: default
+Owner: null
+CreateTime: ......
+LastAccessTime: UNKNOWN
+Retention: 0
+Location: ......
+Table Type: MANAGED_TABLE
+Table Parameters:
+ flink.connector.properties.bootstrap.servers localhost:9092
+ flink.connector.properties.zookeeper.connect localhost:2181
+ flink.connector.topic test
+ flink.connector.type kafka
+ flink.connector.version universal
+ flink.format.type csv
+ flink.generic.table.schema.0.data-type VARCHAR(2147483647)
+ flink.generic.table.schema.0.name name
+ flink.generic.table.schema.1.data-type INT
+ flink.generic.table.schema.1.name age
+ flink.update-mode append
+ is_generic true
+ transient_lastDdlTime ......
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+Time taken: 0.158 seconds, Fetched: 36 row(s)
+
+{% endhighlight %}
+
+
+#### step 5: run Flink SQL to query the Kakfa table
+
+Run a simple select query from Flink SQL Client in a Flink cluster, either standalone or yarn-session.
+
+{% highlight bash %}
+Flink SQL> select * from mykafka;
+
+{% endhighlight %}
+
+
+Produce some more messages in the Kafka topic
+
+{% highlight bash %}
+localhost$ bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
+
+tom,15
+john,21
+kitty,30
+amy,24
+kaiky,18
+
+{% endhighlight %}
+
+
+You should see results produced by Flink in SQL Client now, as:
+
+
+{% highlight bash %}
+ SQL Query Result (Table)
+ Refresh: 1 s Page: Last of 1
+
+ name age
+ tom 15
+ john 21
+ kitty 30
+ amy 24
+ kaiky 18
+
+{% endhighlight %}
+
+## Supported Types
+
+`HiveCatalog` supports all Flink types for generic tables.
+
+For Hive-compatible tables, `HiveCatalog` needs to map Flink data types to corresponding Hive types as described in
+the following table:
+
+
+
+
+
Flink Data Type
+
Hive Data Type
+
+
+
+
+
CHAR(p)
+
CHAR(p)
+
+
+
VARCHAR(p)
+
VARCHAR(p)
+
+
+
STRING
+
STRING
+
+
+
BOOLEAN
+
BOOLEAN
+
+
+
TINYINT
+
TINYINT
+
+
+
SMALLINT
+
SMALLINT
+
+
+
INT
+
INT
+
+
+
BIGINT
+
LONG
+
+
+
FLOAT
+
FLOAT
+
+
+
DOUBLE
+
DOUBLE
+
+
+
DECIMAL(p, s)
+
DECIMAL(p, s)
+
+
+
DATE
+
DATE
+
+
+
TIMESTAMP(9)
+
TIMESTAMP
+
+
+
BYTES
+
BINARY
+
+
+
ARRAY<T>
+
LIST<T>
+
+
+
MAP
+
MAP
+
+
+
ROW
+
STRUCT
+
+
+
+
+Something to note about the type mapping:
+* Hive's `CHAR(p)` has a maximum length of 255
+* Hive's `VARCHAR(p)` has a maximum length of 65535
+* Hive's `MAP` only supports primitive key types while Flink's `MAP` can be any data type
+* Hive's `UNION` type is not supported
+* Hive's `TIMESTAMP` always has precision 9 and doesn't support other precisions. Hive UDFs, on the other hand, can process `TIMESTAMP` values with a precision <= 9.
+* Hive doesn't support Flink's `TIMESTAMP_WITH_TIME_ZONE`, `TIMESTAMP_WITH_LOCAL_TIME_ZONE`, and `MULTISET`
+* Flink's `INTERVAL` type cannot be mapped to Hive `INTERVAL` type yet
diff --git a/docs/dev/table/hive/hive_catalog.zh.md b/docs/dev/table/hive/hive_catalog.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..92b05cbbcbc2ebcba0b820e7808756dd88517b3d
--- /dev/null
+++ b/docs/dev/table/hive/hive_catalog.zh.md
@@ -0,0 +1,392 @@
+---
+title: "HiveCatalog"
+nav-parent_id: hive_tableapi
+nav-pos: 1
+---
+
+
+Hive Metastore has evolved into the de facto metadata hub over the years in Hadoop ecosystem. Many companies have a single
+Hive Metastore service instance in their production to manage all of their metadata, either Hive metadata or non-Hive metadata,
+ as the source of truth.
+
+For users who have both Hive and Flink deployments, `HiveCatalog` enables them to use Hive Metastore to manage Flink's metadata.
+
+For users who have just Flink deployment, `HiveCatalog` is the only persistent catalog provided out-of-box by Flink.
+Without a persistent catalog, users using [Flink SQL CREATE DDL]({{ site.baseurl }}/dev/table/sql/create.html) have to repeatedly
+create meta-objects like a Kafka table in each session, which wastes a lot of time. `HiveCatalog` fills this gap by empowering
+users to create tables and other meta-objects only once, and reference and manage them with convenience later on across sessions.
+
+
+## Set up HiveCatalog
+
+### Dependencies
+
+Setting up a `HiveCatalog` in Flink requires the same [dependencies]({{ site.baseurl }}/dev/table/hive/#dependencies)
+as those of an overall Flink-Hive integration.
+
+### Configuration
+
+Setting up a `HiveCatalog` in Flink requires the same [configuration]({{ site.baseurl }}/dev/table/hive/#connecting-to-hive)
+as those of an overall Flink-Hive integration.
+
+
+## How to use HiveCatalog
+
+Once configured properly, `HiveCatalog` should just work out of box. Users can create Flink meta-objects with DDL, and shoud
+see them immediately afterwards.
+
+`HiveCatalog` can be used to handle two kinds of tables: Hive-compatible tables and generic tables. Hive-compatible tables
+are those stored in a Hive-compatible way, in terms of both metadata and data in the storage layer. Therefore, Hive-compatible tables
+created via Flink can be queried from Hive side.
+
+Generic tables, on the other hand, are specific to Flink. When creating generic tables with `HiveCatalog`, we're just using
+HMS to persist the metadata. While these tables are visible to Hive, it's unlikely Hive is able to understand
+the metadata. And therefore using such tables in Hive leads to undefined behavior.
+
+Flink uses the property '*is_generic*' to tell whether a table is Hive-compatible or generic. When creating a table with
+`HiveCatalog`, it's by default considered generic. If you'd like to create a Hive-compatible table, make sure to set
+`is_generic` to false in your table properties.
+
+As stated above, generic tables shouldn't be used from Hive. In Hive CLI, you can call `DESCRIBE FORMATTED` for a table and
+decide whether it's generic or not by checking the `is_generic` property. Generic tables will have `is_generic=true`.
+
+### Example
+
+We will walk through a simple example here.
+
+#### step 1: set up a Hive Metastore
+
+Have a Hive Metastore running.
+
+Here, we set up a local Hive Metastore and our `hive-site.xml` file in local path `/opt/hive-conf/hive-site.xml`.
+We have some configs like the following:
+
+{% highlight xml %}
+
+
+
+ javax.jdo.option.ConnectionURL
+ jdbc:mysql://localhost/metastore?createDatabaseIfNotExist=true
+ metadata is stored in a MySQL server
+
+
+
+ javax.jdo.option.ConnectionDriverName
+ com.mysql.jdbc.Driver
+ MySQL JDBC driver class
+
+
+
+ javax.jdo.option.ConnectionUserName
+ ...
+ user name for connecting to mysql server
+
+
+
+ javax.jdo.option.ConnectionPassword
+ ...
+ password for connecting to mysql server
+
+
+
+ hive.metastore.uris
+ thrift://localhost:9083
+ IP address (or fully-qualified domain name) and port of the metastore host
+
+
+
+ hive.metastore.schema.verification
+ true
+
+
+
+{% endhighlight %}
+
+
+Test connection to the HMS with Hive Cli. Running some commands, we can see we have a database named `default` and there's no table in it.
+
+
+{% highlight bash %}
+
+hive> show databases;
+OK
+default
+Time taken: 0.032 seconds, Fetched: 1 row(s)
+
+hive> show tables;
+OK
+Time taken: 0.028 seconds, Fetched: 0 row(s)
+{% endhighlight %}
+
+
+#### step 2: configure Flink cluster and SQL CLI
+
+Add all Hive dependencies to `/lib` dir in Flink distribution, and modify SQL CLI's yaml config file `sql-cli-defaults.yaml` as following:
+
+{% highlight yaml %}
+
+execution:
+ planner: blink
+ type: streaming
+ ...
+ current-catalog: myhive # set the HiveCatalog as the current catalog of the session
+ current-database: mydatabase
+
+catalogs:
+ - name: myhive
+ type: hive
+ hive-conf-dir: /opt/hive-conf # contains hive-site.xml
+ hive-version: 2.3.4
+{% endhighlight %}
+
+
+#### step 3: set up a Kafka cluster
+
+Bootstrap a local Kafka 2.3.0 cluster with a topic named "test", and produce some simple data to the topic as tuple of name and age.
+
+{% highlight bash %}
+
+localhost$ bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
+>tom,15
+>john,21
+
+{% endhighlight %}
+
+
+These message can be seen by starting a Kafka console consumer.
+
+{% highlight bash %}
+localhost$ bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
+
+tom,15
+john,21
+
+{% endhighlight %}
+
+
+#### step 4: start SQL Client, and create a Kafka table with Flink SQL DDL
+
+Start Flink SQL Client, create a simple Kafka 2.3.0 table via DDL, and verify its schema.
+
+{% highlight bash %}
+
+Flink SQL> CREATE TABLE mykafka (name String, age Int) WITH (
+ 'connector.type' = 'kafka',
+ 'connector.version' = 'universal',
+ 'connector.topic' = 'test',
+ 'connector.properties.zookeeper.connect' = 'localhost:2181',
+ 'connector.properties.bootstrap.servers' = 'localhost:9092',
+ 'format.type' = 'csv',
+ 'update-mode' = 'append'
+);
+[INFO] Table has been created.
+
+Flink SQL> DESCRIBE mykafka;
+root
+ |-- name: STRING
+ |-- age: INT
+
+{% endhighlight %}
+
+Verify the table is also visible to Hive via Hive Cli, and note that the table has property `is_generic=true`:
+
+{% highlight bash %}
+hive> show tables;
+OK
+mykafka
+Time taken: 0.038 seconds, Fetched: 1 row(s)
+
+hive> describe formatted mykafka;
+OK
+# col_name data_type comment
+
+
+# Detailed Table Information
+Database: default
+Owner: null
+CreateTime: ......
+LastAccessTime: UNKNOWN
+Retention: 0
+Location: ......
+Table Type: MANAGED_TABLE
+Table Parameters:
+ flink.connector.properties.bootstrap.servers localhost:9092
+ flink.connector.properties.zookeeper.connect localhost:2181
+ flink.connector.topic test
+ flink.connector.type kafka
+ flink.connector.version universal
+ flink.format.type csv
+ flink.generic.table.schema.0.data-type VARCHAR(2147483647)
+ flink.generic.table.schema.0.name name
+ flink.generic.table.schema.1.data-type INT
+ flink.generic.table.schema.1.name age
+ flink.update-mode append
+ is_generic true
+ transient_lastDdlTime ......
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+Time taken: 0.158 seconds, Fetched: 36 row(s)
+
+{% endhighlight %}
+
+
+#### step 5: run Flink SQL to query the Kakfa table
+
+Run a simple select query from Flink SQL Client in a Flink cluster, either standalone or yarn-session.
+
+{% highlight bash %}
+Flink SQL> select * from mykafka;
+
+{% endhighlight %}
+
+
+Produce some more messages in the Kafka topic
+
+{% highlight bash %}
+localhost$ bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
+
+tom,15
+john,21
+kitty,30
+amy,24
+kaiky,18
+
+{% endhighlight %}
+
+
+You should see results produced by Flink in SQL Client now, as:
+
+
+{% highlight bash %}
+ SQL Query Result (Table)
+ Refresh: 1 s Page: Last of 1
+
+ name age
+ tom 15
+ john 21
+ kitty 30
+ amy 24
+ kaiky 18
+
+{% endhighlight %}
+
+## Supported Types
+
+`HiveCatalog` supports all Flink types for generic tables.
+
+For Hive-compatible tables, `HiveCatalog` needs to map Flink data types to corresponding Hive types as described in
+the following table:
+
+
+
+
+
Flink Data Type
+
Hive Data Type
+
+
+
+
+
CHAR(p)
+
CHAR(p)
+
+
+
VARCHAR(p)
+
VARCHAR(p)
+
+
+
STRING
+
STRING
+
+
+
BOOLEAN
+
BOOLEAN
+
+
+
TINYINT
+
TINYINT
+
+
+
SMALLINT
+
SMALLINT
+
+
+
INT
+
INT
+
+
+
BIGINT
+
LONG
+
+
+
FLOAT
+
FLOAT
+
+
+
DOUBLE
+
DOUBLE
+
+
+
DECIMAL(p, s)
+
DECIMAL(p, s)
+
+
+
DATE
+
DATE
+
+
+
TIMESTAMP(9)
+
TIMESTAMP
+
+
+
BYTES
+
BINARY
+
+
+
ARRAY<T>
+
LIST<T>
+
+
+
MAP
+
MAP
+
+
+
ROW
+
STRUCT
+
+
+
+
+Something to note about the type mapping:
+* Hive's `CHAR(p)` has a maximum length of 255
+* Hive's `VARCHAR(p)` has a maximum length of 65535
+* Hive's `MAP` only supports primitive key types while Flink's `MAP` can be any data type
+* Hive's `UNION` type is not supported
+* Hive's `TIMESTAMP` always has precision 9 and doesn't support other precisions. Hive UDFs, on the other hand, can process `TIMESTAMP` values with a precision <= 9.
+* Hive doesn't support Flink's `TIMESTAMP_WITH_TIME_ZONE`, `TIMESTAMP_WITH_LOCAL_TIME_ZONE`, and `MULTISET`
+* Flink's `INTERVAL` type cannot be mapped to Hive `INTERVAL` type yet
diff --git a/docs/dev/table/hive/hive_functions.md b/docs/dev/table/hive/hive_functions.md
index 8f1894f0a7b3a2fe4e9b801a76f3d63841fbd149..e2b7ac51b6b6ca255f4f665ec3fefe721f65fd72 100644
--- a/docs/dev/table/hive/hive_functions.md
+++ b/docs/dev/table/hive/hive_functions.md
@@ -1,5 +1,5 @@
---
-title: "Hive Functions"
+title: "Hive functions"
nav-parent_id: hive_tableapi
nav-pos: 3
---
@@ -22,6 +22,46 @@ specific language governing permissions and limitations
under the License.
-->
+## Use Hive Built-in Functions via HiveModule
+
+The `HiveModule` provides Hive built-in functions as Flink system (built-in) functions to Flink SQL and Table API users.
+
+For detailed information, please refer to [HiveModule]({{ site.baseurl }}/dev/table/modules.html#hivemodule).
+
+
+
+{% highlight java %}
+
+String name = "myhive";
+String version = "2.3.4";
+
+tableEnv.loadModue(name, new HiveModule(version));
+{% endhighlight %}
+
+
+{% highlight scala %}
+
+val name = "myhive"
+val version = "2.3.4"
+
+tableEnv.loadModue(name, new HiveModule(version));
+{% endhighlight %}
+
+
+* NOTE that some Hive built-in functions in older versions have [thread safety issues](https://issues.apache.org/jira/browse/HIVE-16183).
+We recommend users patch their own Hive to fix them.
+
## Hive User Defined Functions
Users can use their existing Hive User Defined Functions in Flink.
@@ -154,14 +194,3 @@ Then, users can use them in SQL as:
Flink SQL> select mygenericudf(myudf(name), 1) as a, mygenericudf(myudf(age), 1) as b, s from mysourcetable, lateral table(myudtf(name, 1)) as T(s);
{% endhighlight %}
-
-
-### Limitations
-
-Hive built-in functions are currently not supported out of box in Flink. To use Hive built-in functions, users must register them manually in Hive Metastore first.
-
-Support for Hive functions has only been tested for Flink batch in Blink planner.
-
-Hive functions currently cannot be used across catalogs in Flink.
-
-Please reference to [Hive]({{ site.baseurl }}/dev/table/hive/index.html) for data type limitations.
diff --git a/docs/dev/table/hive/hive_functions.zh.md b/docs/dev/table/hive/hive_functions.zh.md
index 8f1894f0a7b3a2fe4e9b801a76f3d63841fbd149..e2b7ac51b6b6ca255f4f665ec3fefe721f65fd72 100644
--- a/docs/dev/table/hive/hive_functions.zh.md
+++ b/docs/dev/table/hive/hive_functions.zh.md
@@ -1,5 +1,5 @@
---
-title: "Hive Functions"
+title: "Hive functions"
nav-parent_id: hive_tableapi
nav-pos: 3
---
@@ -22,6 +22,46 @@ specific language governing permissions and limitations
under the License.
-->
+## Use Hive Built-in Functions via HiveModule
+
+The `HiveModule` provides Hive built-in functions as Flink system (built-in) functions to Flink SQL and Table API users.
+
+For detailed information, please refer to [HiveModule]({{ site.baseurl }}/dev/table/modules.html#hivemodule).
+
+
+
+{% highlight java %}
+
+String name = "myhive";
+String version = "2.3.4";
+
+tableEnv.loadModue(name, new HiveModule(version));
+{% endhighlight %}
+
+
+{% highlight scala %}
+
+val name = "myhive"
+val version = "2.3.4"
+
+tableEnv.loadModue(name, new HiveModule(version));
+{% endhighlight %}
+
+
+* NOTE that some Hive built-in functions in older versions have [thread safety issues](https://issues.apache.org/jira/browse/HIVE-16183).
+We recommend users patch their own Hive to fix them.
+
## Hive User Defined Functions
Users can use their existing Hive User Defined Functions in Flink.
@@ -154,14 +194,3 @@ Then, users can use them in SQL as:
Flink SQL> select mygenericudf(myudf(name), 1) as a, mygenericudf(myudf(age), 1) as b, s from mysourcetable, lateral table(myudtf(name, 1)) as T(s);
{% endhighlight %}
-
-
-### Limitations
-
-Hive built-in functions are currently not supported out of box in Flink. To use Hive built-in functions, users must register them manually in Hive Metastore first.
-
-Support for Hive functions has only been tested for Flink batch in Blink planner.
-
-Hive functions currently cannot be used across catalogs in Flink.
-
-Please reference to [Hive]({{ site.baseurl }}/dev/table/hive/index.html) for data type limitations.
diff --git a/docs/dev/table/hive/index.md b/docs/dev/table/hive/index.md
index ad2489c7c8efc17fe6570c4a0634cbed400634df..42e825633f23fba0f5946b050d70c18042cafe63 100644
--- a/docs/dev/table/hive/index.md
+++ b/docs/dev/table/hive/index.md
@@ -1,9 +1,8 @@
---
-title: "Hive"
+title: "Hive Integration"
nav-id: hive_tableapi
nav-parent_id: tableapi
-nav-pos: 110
-is_beta: true
+nav-pos: 100
nav-show_overview: true
---
+{% endhighlight %}
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.6.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-metastore-1.0.0.jar
+ hive-exec-1.0.0.jar
+ libfb303-0.9.0.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3-nohive.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
-
+{% endhighlight %}
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.6.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-metastore-1.1.0.jar
+ hive-exec-1.1.0.jar
+ libfb303-0.9.2.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3-nohive.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
-
-
- org.apache.hive
- hive-exec
- 2.3.4
-
{% endhighlight %}
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.6.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-metastore-1.2.1.jar
+ hive-exec-1.2.1.jar
+ libfb303-0.9.2.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3-nohive.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.7.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-2.0.0.jar
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.7.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-2.1.0.jar
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.7.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-2.2.0.jar
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.8.3-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-3.1.0.jar
+ libfb303-0.9.3.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+{% endhighlight %}
+
+
+
+
+If you are building your own program, you need the following dependencies in your mvn file.
+It's recommended not to include these dependencies in the resulting jar file.
+You're supposed to add dependencies as stated above at runtime.
+
{% highlight xml %}
+
org.apache.flinkflink-connector-hive{{ site.scala_version_suffix }}
@@ -100,175 +295,96 @@ To integrate with Hive, users need the following dependencies in their project.
provided
-
-
org.apache.flink
- flink-hadoop-compatibility{{ site.scala_version_suffix }}
+ flink-table-api-java-bridge{{ site.scala_version_suffix }}{{site.version}}provided
-
-
-
- org.apache.flink
- flink-shaded-hadoop-2-uber
- 2.6.5-{{ site.shaded_version }}
- provided
-
-
-
-
- org.apache.hive
- hive-metastore
- 1.2.1
-
-
+
org.apache.hivehive-exec
- 1.2.1
-
-
-
- org.apache.thrift
- libfb303
- 0.9.3
+ ${hive.version}
+ provided
{% endhighlight %}
-
-
## Connecting To Hive
-Connect to an existing Hive installation using the Hive [Catalog]({{ site.baseurl }}/dev/table/catalogs.html) through the table environment or YAML configuration.
+Connect to an existing Hive installation using the [catalog interface]({{ site.baseurl }}/dev/table/catalogs.html)
+and [HiveCatalog]({{ site.baseurl }}/dev/table/hive/hive_catalog.html) through the table environment or YAML configuration.
+
+If the `hive-conf/hive-site.xml` file is stored in remote storage system, users should download
+the hive configuration file to their local environment first.
+
+Please note while HiveCatalog doesn't require a particular planner, reading/writing Hive tables only works with blink planner.
+Therefore it's highly recommended that you use blink planner when connecting to your Hive warehouse.
+
+Take Hive version 2.3.4 for example:
{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build();
+TableEnvironment tableEnv = TableEnvironment.create(settings);
+
String name = "myhive";
String defaultDatabase = "mydatabase";
-String hiveConfDir = "/opt/hive-conf";
-String version = "2.3.4"; // or 1.2.1
+String hiveConfDir = "/opt/hive-conf"; // a local path
+String version = "2.3.4";
HiveCatalog hive = new HiveCatalog(name, defaultDatabase, hiveConfDir, version);
tableEnv.registerCatalog("myhive", hive);
+
+// set the HiveCatalog as the current catalog of the session
+tableEnv.useCatalog("myhive");
{% endhighlight %}
{% highlight scala %}
+val settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build()
+val tableEnv = TableEnvironment.create(settings)
+
val name = "myhive"
val defaultDatabase = "mydatabase"
-val hiveConfDir = "/opt/hive-conf"
-val version = "2.3.4" // or 1.2.1
+val hiveConfDir = "/opt/hive-conf" // a local path
+val version = "2.3.4"
val hive = new HiveCatalog(name, defaultDatabase, hiveConfDir, version)
tableEnv.registerCatalog("myhive", hive)
+
+// set the HiveCatalog as the current catalog of the session
+tableEnv.useCatalog("myhive")
{% endhighlight %}
{% highlight yaml %}
+
+execution:
+ planner: blink
+ ...
+ current-catalog: myhive # set the HiveCatalog as the current catalog of the session
+ current-database: mydatabase
+
catalogs:
- name: myhive
type: hive
- property-version: 1
hive-conf-dir: /opt/hive-conf
- hive-version: 2.3.4 # or 1.2.1
+ hive-version: 2.3.4
{% endhighlight %}
-## Supported Types
-
-Currently `HiveCatalog` supports most Flink data types with the following mapping:
-
-
-
-
-
Flink Data Type
-
Hive Data Type
-
-
-
-
-
CHAR(p)
-
CHAR(p)
-
-
-
VARCHAR(p)
-
VARCHAR(p)
-
-
-
STRING
-
STRING
-
-
-
BOOLEAN
-
BOOLEAN
-
-
-
TINYINT
-
TINYINT
-
-
-
SMALLINT
-
SMALLINT
-
-
-
INT
-
INT
-
-
-
BIGINT
-
LONG
-
-
-
FLOAT
-
FLOAT
-
-
-
DOUBLE
-
DOUBLE
-
-
-
DECIMAL(p, s)
-
DECIMAL(p, s)
-
-
-
DATE
-
DATE
-
-
-
BYTES
-
BINARY
-
-
-
ARRAY<T>
-
LIST<T>
-
-
-
MAP
-
MAP
-
-
-
ROW
-
STRUCT
-
-
-
-
-### Limitations
-
-The following limitations in Hive's data types impact the mapping between Flink and Hive:
-
-* `CHAR(p)` has a maximum length of 255
-* `VARCHAR(p)` has a maximum length of 65535
-* Hive's `MAP` only supports primitive key types while Flink's `MAP` can be any data type
-* Hive's `UNION` type is not supported
-* Flink's `INTERVAL` type cannot be mapped to Hive `INTERVAL` type
-* Flink's `TIMESTAMP_WITH_TIME_ZONE` and `TIMESTAMP_WITH_LOCAL_TIME_ZONE` are not supported by Hive
-* Flink's `TIMESTAMP_WITHOUT_TIME_ZONE` type cannot be mapped to Hive's `TIMESTAMP` type due to precision difference.
-* Flink's `MULTISET` is not supported by Hive
+
+## DDL
+
+DDL to create Hive tables, views, partitions, functions within Flink will be supported soon.
+
+## DML
+
+Flink supports DML writing to Hive tables. Please refer to details in [Reading & Writing Hive Tables]({{ site.baseurl }}/dev/table/hive/read_write_hive.html)
+
diff --git a/docs/dev/table/hive/index.zh.md b/docs/dev/table/hive/index.zh.md
index 9beedf330d63fbfa9f87935423afc753a208ccfe..925dce1326bd37ae056d87ede40c56255e9b2c10 100644
--- a/docs/dev/table/hive/index.zh.md
+++ b/docs/dev/table/hive/index.zh.md
@@ -1,9 +1,8 @@
---
-title: "Hive"
+title: "Hive Integration"
nav-id: hive_tableapi
nav-parent_id: tableapi
nav-pos: 100
-is_beta: true
nav-show_overview: true
---
+{% endhighlight %}
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.6.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-metastore-1.0.0.jar
+ hive-exec-1.0.0.jar
+ libfb303-0.9.0.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3-nohive.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
-
+{% endhighlight %}
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.6.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-metastore-1.1.0.jar
+ hive-exec-1.1.0.jar
+ libfb303-0.9.2.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3-nohive.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
-
-
- org.apache.hive
- hive-exec
- 2.3.4
-
{% endhighlight %}
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.6.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-metastore-1.2.1.jar
+ hive-exec-1.2.1.jar
+ libfb303-0.9.2.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3-nohive.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.7.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-2.0.0.jar
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.7.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-2.1.0.jar
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.7.5-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-2.2.0.jar
+
+ // Orc dependencies -- required by the ORC vectorized optimizations
+ orc-core-1.4.3.jar
+ aircompressor-0.8.jar // transitive dependency of orc-core
+
+{% endhighlight %}
+
+
+
+{% highlight txt %}
+/flink-{{ site.version }}
+ /lib
+
+ // Flink's Hive connector
+ flink-connector-hive{{ site.scala_version_suffix }}-{{ site.version }}.jar
+
+ // Hadoop dependencies
+ // You can pick a pre-built Hadoop uber jar provided by Flink, alternatively
+ // you can use your own hadoop jars. Either way, make sure it's compatible with your Hadoop
+ // cluster and the Hive version you're using.
+ flink-shaded-hadoop-2-uber-2.8.3-{{ site.shaded_version }}.jar
+
+ // Hive dependencies
+ hive-exec-3.1.0.jar
+ libfb303-0.9.3.jar // libfb303 is not packed into hive-exec in some versions, need to add it separately
+
+{% endhighlight %}
+
+
+
+
+If you are building your own program, you need the following dependencies in your mvn file.
+It's recommended not to include these dependencies in the resulting jar file.
+You're supposed to add dependencies as stated above at runtime.
+
{% highlight xml %}
+
org.apache.flinkflink-connector-hive{{ site.scala_version_suffix }}
@@ -100,175 +295,95 @@ To integrate with Hive, users need the following dependencies in their project.
provided
-
-
org.apache.flink
- flink-hadoop-compatibility{{ site.scala_version_suffix }}
+ flink-table-api-java-bridge{{ site.scala_version_suffix }}{{site.version}}provided
-
-
-
- org.apache.flink
- flink-shaded-hadoop-2-uber
- 2.6.5-{{ site.shaded_version }}
- provided
-
-
-
-
- org.apache.hive
- hive-metastore
- 1.2.1
-
-
+
org.apache.hivehive-exec
- 1.2.1
-
-
-
- org.apache.thrift
- libfb303
- 0.9.3
+ ${hive.version}
+ provided
{% endhighlight %}
-
-
## Connecting To Hive
-Connect to an existing Hive installation using the Hive [Catalog]({{ site.baseurl }}/dev/table/catalogs.html) through the table environment or YAML configuration.
+Connect to an existing Hive installation using the [catalog interface]({{ site.baseurl }}/dev/table/catalogs.html)
+and [HiveCatalog]({{ site.baseurl }}/dev/table/hive/hive_catalog.html) through the table environment or YAML configuration.
+
+If the `hive-conf/hive-site.xml` file is stored in remote storage system, users should download
+the hive configuration file to their local environment first.
+
+Please note while HiveCatalog doesn't require a particular planner, reading/writing Hive tables only works with blink planner.
+Therefore it's highly recommended that you use blink planner when connecting to your Hive warehouse.
+
+Take Hive version 2.3.4 for example:
{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build();
+TableEnvironment tableEnv = TableEnvironment.create(settings);
+
String name = "myhive";
String defaultDatabase = "mydatabase";
-String hiveConfDir = "/opt/hive-conf";
-String version = "2.3.4"; // or 1.2.1
+String hiveConfDir = "/opt/hive-conf"; // a local path
+String version = "2.3.4";
HiveCatalog hive = new HiveCatalog(name, defaultDatabase, hiveConfDir, version);
tableEnv.registerCatalog("myhive", hive);
+
+// set the HiveCatalog as the current catalog of the session
+tableEnv.useCatalog("myhive");
{% endhighlight %}
{% highlight scala %}
+val settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build()
+val tableEnv = TableEnvironment.create(settings)
+
val name = "myhive"
val defaultDatabase = "mydatabase"
-val hiveConfDir = "/opt/hive-conf"
-val version = "2.3.4" // or 1.2.1
+val hiveConfDir = "/opt/hive-conf" // a local path
+val version = "2.3.4"
val hive = new HiveCatalog(name, defaultDatabase, hiveConfDir, version)
tableEnv.registerCatalog("myhive", hive)
+
+// set the HiveCatalog as the current catalog of the session
+tableEnv.useCatalog("myhive")
{% endhighlight %}
{% highlight yaml %}
+
+execution:
+ planner: blink
+ ...
+ current-catalog: myhive # set the HiveCatalog as the current catalog of the session
+ current-database: mydatabase
+
catalogs:
- name: myhive
type: hive
- property-version: 1
hive-conf-dir: /opt/hive-conf
- hive-version: 2.3.4 # or 1.2.1
+ hive-version: 2.3.4
{% endhighlight %}
-## Supported Types
-
-Currently `HiveCatalog` supports most Flink data types with the following mapping:
-
-
-
-
-
Flink Data Type
-
Hive Data Type
-
-
-
-
-
CHAR(p)
-
CHAR(p)
-
-
-
VARCHAR(p)
-
VARCHAR(p)
-
-
-
STRING
-
STRING
-
-
-
BOOLEAN
-
BOOLEAN
-
-
-
TINYINT
-
TINYINT
-
-
-
SMALLINT
-
SMALLINT
-
-
-
INT
-
INT
-
-
-
BIGINT
-
LONG
-
-
-
FLOAT
-
FLOAT
-
-
-
DOUBLE
-
DOUBLE
-
-
-
DECIMAL(p, s)
-
DECIMAL(p, s)
-
-
-
DATE
-
DATE
-
-
-
BYTES
-
BINARY
-
-
-
ARRAY<T>
-
LIST<T>
-
-
-
MAP
-
MAP
-
-
-
ROW
-
STRUCT
-
-
-
-
-### Limitations
-
-The following limitations in Hive's data types impact the mapping between Flink and Hive:
-
-* `CHAR(p)` has a maximum length of 255
-* `VARCHAR(p)` has a maximum length of 65535
-* Hive's `MAP` only supports primitive key types while Flink's `MAP` can be any data type
-* Hive's `UNION` type is not supported
-* Flink's `INTERVAL` type cannot be mapped to Hive `INTERVAL` type
-* Flink's `TIMESTAMP_WITH_TIME_ZONE` and `TIMESTAMP_WITH_LOCAL_TIME_ZONE` are not supported by Hive
-* Flink's `TIMESTAMP_WITHOUT_TIME_ZONE` type cannot be mapped to Hive's `TIMESTAMP` type due to precision difference.
-* Flink's `MULTISET` is not supported by Hive
+
+## DDL
+
+DDL to create Hive tables, views, partitions, functions within Flink will be supported soon.
+
+## DML
+
+Flink supports DML writing to Hive tables. Please refer to details in [Reading & Writing Hive Tables]({{ site.baseurl }}/dev/table/hive/read_write_hive.html)
diff --git a/docs/dev/table/hive/read_write_hive.md b/docs/dev/table/hive/read_write_hive.md
index 4a48060edb81d28de32cb4337c3fce355c7d375f..07ee3b34d579cdcf222c2e74b85c1af1eaeb11cc 100644
--- a/docs/dev/table/hive/read_write_hive.md
+++ b/docs/dev/table/hive/read_write_hive.md
@@ -22,8 +22,9 @@ specific language governing permissions and limitations
under the License.
-->
-Using the `HiveCatalog` and Flink's connector to Hive, Flink can read and write from Hive data as an alternative to Hive's batch engine. Be sure to follow the instructions to include the correct [dependencies]({{ site.baseurl }}/dev/table/hive/#depedencies) in your application.
-
+Using the `HiveCatalog` and Flink's connector to Hive, Flink can read and write from Hive data as an alternative to Hive's batch engine.
+Be sure to follow the instructions to include the correct [dependencies]({{ site.baseurl }}/dev/table/hive/#depedencies) in your application.
+And please also note that Hive connector only works with blink planner.
* This will be replaced by the TOC
{:toc}
@@ -91,7 +92,7 @@ root
|-- name: value
|-- type: DOUBLE
-
+# ------ Select from hive table or hive view ------
Flink SQL> SELECT * FROM mytable;
name value
@@ -110,22 +111,85 @@ __________ __________
{% endhighlight %}
+### Querying Hive views
+
+If you need to query Hive views, please note:
+
+1. You have to use the Hive catalog as your current catalog before you can query views in that catalog. It can be done by either `tableEnv.useCatalog(...)` in Table API or `USE CATALOG ...` in SQL Client.
+2. Hive and Flink SQL have different syntax, e.g. different reserved keywords and literals. Make sure the view's query is compatible with Flink grammar.
+
## Writing To Hive
-Similarly, data can be written into hive using an `INSERT INTO` clause.
+Similarly, data can be written into hive using an `INSERT` clause.
+
+Consider there is an example table named "mytable" with two columns: name and age, in string and int type.
+
+{% highlight bash %}
+# ------ INSERT INTO will append to the table or partition, keeping the existing data intact ------
+Flink SQL> INSERT INTO mytable SELECT 'Tom', 25;
+
+# ------ INSERT OVERWRITE will overwrite any existing data in the table or partition ------
+Flink SQL> INSERT OVERWRITE mytable SELECT 'Tom', 25;
+{% endhighlight %}
+
+We support partitioned table too, Consider there is a partitioned table named myparttable with four columns: name, age, my_type and my_date, in types ...... my_type and my_date are the partition keys.
+
+{% highlight bash %}
+# ------ Insert with static partition ------
+Flink SQL> INSERT OVERWRITE myparttable PARTITION (my_type='type_1', my_date='2019-08-08') SELECT 'Tom', 25;
+
+# ------ Insert with dynamic partition ------
+Flink SQL> INSERT OVERWRITE myparttable SELECT 'Tom', 25, 'type_1', '2019-08-08';
+
+# ------ Insert with static(my_type) and dynamic(my_date) partition ------
+Flink SQL> INSERT OVERWRITE myparttable PARTITION (my_type='type_1') SELECT 'Tom', 25, '2019-08-08';
+{% endhighlight %}
+
+
+## Formats
+
+We have tested on the following of table storage formats: text, csv, SequenceFile, ORC, and Parquet.
+
+
+## Optimizations
+
+### Partition Pruning
+
+Flink uses partition pruning as a performance optimization to limits the number of files and partitions
+that Flink reads when querying Hive tables. When your data is partitioned, Flink only reads a subset of the partitions in
+a Hive table when a query matches certain filter criteria.
+
+### Projection Pushdown
+
+Flink leverages projection pushdown to minimize data transfer between Flink and Hive tables by omitting
+unnecessary fields from table scans.
+
+It is especially beneficial when a table contains many columns.
+
+### Limit Pushdown
+
+For queries with LIMIT clause, Flink will limit the number of output records wherever possible to minimize the
+amount of data transferred across network.
+
+### ORC Vectorized Optimization upon Read
+
+Optimization is used automatically when the following conditions are met:
+
+- Columns without complex data type, like hive types: List, Map, Struct, Union.
+
+This feature is turned on by default. If there is a problem, you can use this config option to close ORC Vectorized Optimization:
{% highlight bash %}
-Flink SQL> INSERT INTO mytable (name, value) VALUES ('Tom', 4.72);
+table.exec.hive.fallback-mapred-reader=true
{% endhighlight %}
-### Limitations
-The following is a list of major limitations of the Hive connector. And we're actively working to close these gaps.
+## Roadmap
+
+We are planning and actively working on supporting features like
+
+- ACID tables
+- bucketed tables
+- more formats
-1. INSERT OVERWRITE is not supported.
-2. Inserting into partitioned tables is not supported.
-3. ACID tables are not supported.
-4. Bucketed tables are not supported.
-5. Some data types are not supported. See the [limitations]({{ site.baseurl }}/dev/table/hive/#limitations) for details.
-6. Only a limited number of table storage formats have been tested, namely text, SequenceFile, ORC, and Parquet.
-7. Views are not supported.
+Please reach out to the community for more feature request https://flink.apache.org/community.html#mailing-lists
diff --git a/docs/dev/table/hive/read_write_hive.zh.md b/docs/dev/table/hive/read_write_hive.zh.md
index 9b9bf77da730b03665bdf4c2849d0c852a66e39b..07ee3b34d579cdcf222c2e74b85c1af1eaeb11cc 100644
--- a/docs/dev/table/hive/read_write_hive.zh.md
+++ b/docs/dev/table/hive/read_write_hive.zh.md
@@ -22,8 +22,9 @@ specific language governing permissions and limitations
under the License.
-->
-Using the `HiveCatalog` and Flink's connector to Hive, Flink can read and write from Hive data as an alternative to Hive's batch engine. Be sure to follow the instructions to include the correct [dependencies]({{ site.baseurl }}/dev/table/hive/#depedencies) in your application.
-
+Using the `HiveCatalog` and Flink's connector to Hive, Flink can read and write from Hive data as an alternative to Hive's batch engine.
+Be sure to follow the instructions to include the correct [dependencies]({{ site.baseurl }}/dev/table/hive/#depedencies) in your application.
+And please also note that Hive connector only works with blink planner.
* This will be replaced by the TOC
{:toc}
@@ -91,7 +92,7 @@ root
|-- name: value
|-- type: DOUBLE
-
+# ------ Select from hive table or hive view ------
Flink SQL> SELECT * FROM mytable;
name value
@@ -110,22 +111,85 @@ __________ __________
{% endhighlight %}
+### Querying Hive views
+
+If you need to query Hive views, please note:
+
+1. You have to use the Hive catalog as your current catalog before you can query views in that catalog. It can be done by either `tableEnv.useCatalog(...)` in Table API or `USE CATALOG ...` in SQL Client.
+2. Hive and Flink SQL have different syntax, e.g. different reserved keywords and literals. Make sure the view's query is compatible with Flink grammar.
+
## Writing To Hive
-Similarly, data can be written into hive using an `INSERT INTO` clause.
+Similarly, data can be written into hive using an `INSERT` clause.
+
+Consider there is an example table named "mytable" with two columns: name and age, in string and int type.
+
+{% highlight bash %}
+# ------ INSERT INTO will append to the table or partition, keeping the existing data intact ------
+Flink SQL> INSERT INTO mytable SELECT 'Tom', 25;
+
+# ------ INSERT OVERWRITE will overwrite any existing data in the table or partition ------
+Flink SQL> INSERT OVERWRITE mytable SELECT 'Tom', 25;
+{% endhighlight %}
+
+We support partitioned table too, Consider there is a partitioned table named myparttable with four columns: name, age, my_type and my_date, in types ...... my_type and my_date are the partition keys.
+
+{% highlight bash %}
+# ------ Insert with static partition ------
+Flink SQL> INSERT OVERWRITE myparttable PARTITION (my_type='type_1', my_date='2019-08-08') SELECT 'Tom', 25;
+
+# ------ Insert with dynamic partition ------
+Flink SQL> INSERT OVERWRITE myparttable SELECT 'Tom', 25, 'type_1', '2019-08-08';
+
+# ------ Insert with static(my_type) and dynamic(my_date) partition ------
+Flink SQL> INSERT OVERWRITE myparttable PARTITION (my_type='type_1') SELECT 'Tom', 25, '2019-08-08';
+{% endhighlight %}
+
+
+## Formats
+
+We have tested on the following of table storage formats: text, csv, SequenceFile, ORC, and Parquet.
+
+
+## Optimizations
+
+### Partition Pruning
+
+Flink uses partition pruning as a performance optimization to limits the number of files and partitions
+that Flink reads when querying Hive tables. When your data is partitioned, Flink only reads a subset of the partitions in
+a Hive table when a query matches certain filter criteria.
+
+### Projection Pushdown
+
+Flink leverages projection pushdown to minimize data transfer between Flink and Hive tables by omitting
+unnecessary fields from table scans.
+
+It is especially beneficial when a table contains many columns.
+
+### Limit Pushdown
+
+For queries with LIMIT clause, Flink will limit the number of output records wherever possible to minimize the
+amount of data transferred across network.
+
+### ORC Vectorized Optimization upon Read
+
+Optimization is used automatically when the following conditions are met:
+
+- Columns without complex data type, like hive types: List, Map, Struct, Union.
+
+This feature is turned on by default. If there is a problem, you can use this config option to close ORC Vectorized Optimization:
{% highlight bash %}
-Flink SQL> INSERT INTO mytable (name, value) VALUES ('Tom', 4.72);
+table.exec.hive.fallback-mapred-reader=true
{% endhighlight %}
-### Limitations
-The following is a list of major limitations of the Hive connector. And we're actively working to close these gaps.
+## Roadmap
+
+We are planning and actively working on supporting features like
+
+- ACID tables
+- bucketed tables
+- more formats
-1. INSERT OVERWRITE is not supported.
-2. Inserting into partitioned tables is not supported.
-3. ACID tables are not supported.
-4. Bucketed tables are not supported.
-5. Some data types are not supported. See the [limitations]({{ site.baseurl }}/zh/dev/table/hive/#limitations) for details.
-6. Only a limited number of table storage formats have been tested, namely text, SequenceFile, ORC, and Parquet.
-7. Views are not supported.
+Please reach out to the community for more feature request https://flink.apache.org/community.html#mailing-lists
diff --git a/docs/dev/table/hive/scala_shell_hive.md b/docs/dev/table/hive/scala_shell_hive.md
index b9e91556b586c24fcf35ab4358f57ebb35b836b9..3710272cdcf0bcb573abc2a9091d178b9986264d 100644
--- a/docs/dev/table/hive/scala_shell_hive.md
+++ b/docs/dev/table/hive/scala_shell_hive.md
@@ -22,6 +22,8 @@ specific language governing permissions and limitations
under the License.
-->
+NOTE: since blink planner is not well supported in Scala Shell at the moment, it's **NOT** recommended to use Hive connector in Scala Shell.
+
[Flink Scala Shell]({{ site.baseurl }}/ops/scala_shell.html) is a convenient quick way to try flink.
You can use hive in scala shell as well instead of specifying hive dependencies in pom file, packaging your program and submitting it via flink run command.
In order to use hive connector in scala shell, you need to put the following [hive connector dependencies]({{ site.baseurl }}/dev/table/hive/#depedencies) under lib folder of flink dist .
diff --git a/docs/dev/table/hive/scala_shell_hive.zh.md b/docs/dev/table/hive/scala_shell_hive.zh.md
index b9e91556b586c24fcf35ab4358f57ebb35b836b9..3710272cdcf0bcb573abc2a9091d178b9986264d 100644
--- a/docs/dev/table/hive/scala_shell_hive.zh.md
+++ b/docs/dev/table/hive/scala_shell_hive.zh.md
@@ -22,6 +22,8 @@ specific language governing permissions and limitations
under the License.
-->
+NOTE: since blink planner is not well supported in Scala Shell at the moment, it's **NOT** recommended to use Hive connector in Scala Shell.
+
[Flink Scala Shell]({{ site.baseurl }}/ops/scala_shell.html) is a convenient quick way to try flink.
You can use hive in scala shell as well instead of specifying hive dependencies in pom file, packaging your program and submitting it via flink run command.
In order to use hive connector in scala shell, you need to put the following [hive connector dependencies]({{ site.baseurl }}/dev/table/hive/#depedencies) under lib folder of flink dist .
diff --git a/docs/dev/table/index.md b/docs/dev/table/index.md
index 3ef2a7cd7ea3907cae66131045dac92045e47109..165f7de0125f78df0a3379c215106c896208c24c 100644
--- a/docs/dev/table/index.md
+++ b/docs/dev/table/index.md
@@ -111,7 +111,7 @@ Internally, parts of the table ecosystem are implemented in Scala. Therefore, pl
### Extension Dependencies
-If you want to implement a [custom format]({{ site.baseurl }}/dev/table/sourceSinks.html#define-a-tablefactory) for interacting with Kafka or a set of [user-defined functions]({{ site.baseurl }}/dev/table/functions.html), the following dependency is sufficient and can be used for JAR files for the SQL Client:
+If you want to implement a [custom format]({{ site.baseurl }}/dev/table/sourceSinks.html#define-a-tablefactory) for interacting with Kafka or a set of [user-defined functions]({{ site.baseurl }}/dev/table/functions/systemFunctions.html), the following dependency is sufficient and can be used for JAR files for the SQL Client:
{% highlight xml %}
@@ -137,10 +137,10 @@ Where to go next?
* [Concepts & Common API]({{ site.baseurl }}/dev/table/common.html): Shared concepts and APIs of the Table API and SQL.
* [Data Types]({{ site.baseurl }}/dev/table/types.html): Lists pre-defined data types and their properties.
* [Streaming Concepts]({{ site.baseurl }}/dev/table/streaming): Streaming-specific documentation for the Table API or SQL such as configuration of time attributes and handling of updating results.
-* [Connect to External Systems]({{ site.baseurl }}/dev/table/functions.html): Available connectors and formats for reading and writing data to external systems.
+* [Connect to External Systems]({{ site.baseurl }}/dev/table/connect.html): Available connectors and formats for reading and writing data to external systems.
* [Table API]({{ site.baseurl }}/dev/table/tableApi.html): Supported operations and API for the Table API.
-* [SQL]({{ site.baseurl }}/dev/table/sql.html): Supported operations and syntax for SQL.
-* [Built-in Functions]({{ site.baseurl }}/dev/table/functions.html): Supported functions in Table API and SQL.
+* [SQL]({{ site.baseurl }}/dev/table/sql/index.html): Supported operations and syntax for SQL.
+* [Built-in Functions]({{ site.baseurl }}/dev/table/functions/systemFunctions.html): Supported functions in Table API and SQL.
* [SQL Client]({{ site.baseurl }}/dev/table/sqlClient.html): Play around with Flink SQL and submit a table program to a cluster without programming knowledge.
{% top %}
diff --git a/docs/dev/table/index.zh.md b/docs/dev/table/index.zh.md
index 3ef2a7cd7ea3907cae66131045dac92045e47109..8613b117830dc0e2f177b3a22308c8b58f2292cd 100644
--- a/docs/dev/table/index.zh.md
+++ b/docs/dev/table/index.zh.md
@@ -25,41 +25,41 @@ specific language governing permissions and limitations
under the License.
-->
-Apache Flink features two relational APIs - the Table API and SQL - for unified stream and batch processing. The Table API is a language-integrated query API for Scala and Java that allows the composition of queries from relational operators such as selection, filter, and join in a very intuitive way. Flink's SQL support is based on [Apache Calcite](https://calcite.apache.org) which implements the SQL standard. Queries specified in either interface have the same semantics and specify the same result regardless whether the input is a batch input (DataSet) or a stream input (DataStream).
+Apache Flink 有两种关系型 API 来做流批统一处理：Table API 和 SQL。Table API 是用于 Scala 和 Java 语言的查询API，它可以用一种非常直观的方式来组合使用选取、过滤、join 等关系型算子。Flink SQL 是基于 [Apache Calcite](https://calcite.apache.org) 来实现的标准 SQL。这两种 API 中的查询对于批（DataSet）和流（DataStream）的输入有相同的语义，也会产生同样的计算结果。
-The Table API and the SQL interfaces are tightly integrated with each other as well as Flink's DataStream and DataSet APIs. You can easily switch between all APIs and libraries which build upon the APIs. For instance, you can extract patterns from a DataStream using the [CEP library]({{ site.baseurl }}/dev/libs/cep.html) and later use the Table API to analyze the patterns, or you might scan, filter, and aggregate a batch table using a SQL query before running a [Gelly graph algorithm]({{ site.baseurl }}/dev/libs/gelly) on the preprocessed data.
+Table API 和 SQL 两种 API 是紧密集成的，以及 DataStream 和 DataSet API。你可以在这些 API 之间，以及一些基于这些 API 的库之间轻松的切换。比如，你可以先用 [CEP]({{ site.baseurl }}/zh/dev/libs/cep.html) 从 DataStream 中做模式匹配，然后用 Table API 来分析匹配的结果；或者你可以用 SQL 来扫描、过滤、聚合一个批式的表，然后再跑一个 [Gelly 图算法]({{ site.baseurl }}/zh/dev/libs/gelly) 来处理已经预处理好的数据。
-**Please note that the Table API and SQL are not yet feature complete and are being actively developed. Not all operations are supported by every combination of \[Table API, SQL\] and \[stream, batch\] input.**
+**注意：Table API 和 SQL 现在还处于活跃开发阶段，还没有完全实现所有的特性。不是所有的 \[Table API，SQL\] 和 \[流，批\] 的组合都是支持的。**
-Dependency Structure
+依赖图
--------------------
-Starting from Flink 1.9, Flink provides two different planner implementations for evaluating Table & SQL API programs: the Blink planner and the old planner that was available before Flink 1.9. Planners are responsible for
-translating relational operators into an executable, optimized Flink job. Both of the planners come with different optimization rules and runtime classes.
-They may also differ in the set of supported features.
+从1.9开始，Flink 提供了两个 Table Planner 实现来执行 Table API 和 SQL 程序：Blink Planner 和 Old Planner，Old Planner 在1.9之前就已经存在了。
+Planner 的作用主要是把关系型的操作翻译成可执行的、经过优化的 Flink 任务。两种 Planner 所使用的优化规则以及运行时类都不一样。
+它们在支持的功能上也有些差异。
-Attention For production use cases, we recommend the old planner that was present before Flink 1.9 for now.
+注意 对于生产环境，我们建议使用在1.9之前就已经存在的 Old Planner。
-All Table API and SQL components are bundled in the `flink-table` or `flink-table-blink` Maven artifacts.
+所有的 Table API 和 SQL 的代码都在 `flink-table` 或者 `flink-table-blink` Maven artifacts 下。
-The following dependencies are relevant for most projects:
+下面是各个依赖：
-* `flink-table-common`: A common module for extending the table ecosystem by custom functions, formats, etc.
-* `flink-table-api-java`: The Table & SQL API for pure table programs using the Java programming language (in early development stage, not recommended!).
-* `flink-table-api-scala`: The Table & SQL API for pure table programs using the Scala programming language (in early development stage, not recommended!).
-* `flink-table-api-java-bridge`: The Table & SQL API with DataStream/DataSet API support using the Java programming language.
-* `flink-table-api-scala-bridge`: The Table & SQL API with DataStream/DataSet API support using the Scala programming language.
-* `flink-table-planner`: The table program planner and runtime. This was the only planner of Flink before the 1.9 release. It is still the recommended one.
-* `flink-table-planner-blink`: The new Blink planner.
-* `flink-table-runtime-blink`: The new Blink runtime.
-* `flink-table-uber`: Packages the API modules above plus the old planner into a distribution for most Table & SQL API use cases. The uber JAR file `flink-table-*.jar` is located in the `/lib` directory of a Flink release by default.
-* `flink-table-uber-blink`: Packages the API modules above plus the Blink specific modules into a distribution for most Table & SQL API use cases. The uber JAR file `flink-table-blink-*.jar` is located in the `/lib` directory of a Flink release by default.
+* `flink-table-common`: 公共模块，比如自定义函数、格式等需要依赖的。
+* `flink-table-api-java`: Table 和 SQL API，使用 Java 语言编写的，给纯 table 程序使用（还在早期开发阶段，不建议使用）
+* `flink-table-api-scala`: Table 和 SQL API，使用 Scala 语言编写的，给纯 table 程序使用（还在早期开发阶段，不建议使用）
+* `flink-table-api-java-bridge`: Table 和 SQL API 结合 DataStream/DataSet API 一起使用，给 Java 语言使用。
+* `flink-table-api-scala-bridge`: Table 和 SQL API 结合 DataStream/DataSet API 一起使用，给 Scala 语言使用。
+* `flink-table-planner`: table Planner 和运行时。这是在1.9之前 Flink 的唯一的 Planner，现在仍然建议使用这个。
+* `flink-table-planner-blink`: 新的 Blink Planner。
+* `flink-table-runtime-blink`: 新的 Blink 运行时。
+* `flink-table-uber`: 把上述模块以及 Old Planner 打包到一起，可以在大部分 Table & SQL API 场景下使用。打包到一起的 jar 文件 `flink-table-*.jar` 默认会直接放到 Flink 发行版的 `/lib` 目录下。
+* `flink-table-uber-blink`: 把上述模块以及 Blink Planner 打包到一起，可以在大部分 Table & SQL API 场景下使用。打包到一起的 jar 文件 `flink-table-blink-*.jar` 默认会放到 Flink 发行版的 `/lib` 目录下。
-See the [common API](common.html) page for more information about how to switch between the old and new Blink planner in table programs.
+关于如何使用 Old Planner 以及 Blink Planner，可以参考[公共 API](common.html)。
-### Table Program Dependencies
+### Table 程序依赖
-Depending on the target programming language, you need to add the Java or Scala API to a project in order to use the Table API & SQL for defining pipelines:
+取决于你使用的编程语言，选择 Java 或者 Scala API 来构建你的 Table API 和 SQL 程序：
{% highlight xml %}
@@ -78,8 +78,7 @@ Depending on the target programming language, you need to add the Java or Scala
{% endhighlight %}
-Additionally, if you want to run the Table API & SQL programs locally within your IDE, you must add one of the
-following set of modules, depending which planner you want to use:
+除此之外，如果你想在 IDE 本地运行你的程序，你需要添加下面的模块，具体用哪个取决于你使用哪个 Planner：
{% highlight xml %}
@@ -98,7 +97,7 @@ following set of modules, depending which planner you want to use:
{% endhighlight %}
-Internally, parts of the table ecosystem are implemented in Scala. Therefore, please make sure to add the following dependency for both batch and streaming applications:
+内部实现上，部分 table 相关的代码是用 Scala 实现的。所以，下面的依赖也需要添加到你的程序里，不管是批式还是流式的程序：
{% highlight xml %}
@@ -109,9 +108,9 @@ Internally, parts of the table ecosystem are implemented in Scala. Therefore, pl
{% endhighlight %}
-### Extension Dependencies
+### 扩展依赖
-If you want to implement a [custom format]({{ site.baseurl }}/dev/table/sourceSinks.html#define-a-tablefactory) for interacting with Kafka or a set of [user-defined functions]({{ site.baseurl }}/dev/table/functions.html), the following dependency is sufficient and can be used for JAR files for the SQL Client:
+如果你想实现[自定义格式]({{ site.baseurl }}/zh/dev/table/sourceSinks.html#define-a-tablefactory)来解析 Kafka 数据，或者[自定义函数]({{ site.baseurl }}/zh/dev/table/functions/systemFunctions.html)，下面的依赖就足够了，编译出来的 jar 文件可以直接给 SQL Client 使用：
{% highlight xml %}
@@ -122,7 +121,7 @@ If you want to implement a [custom format]({{ site.baseurl }}/dev/table/sourceSi
{% endhighlight %}
-Currently, the module includes extension points for:
+当前，本模块包含以下可以扩展的接口：
- `SerializationSchemaFactory`
- `DeserializationSchemaFactory`
- `ScalarFunction`
@@ -131,16 +130,16 @@ Currently, the module includes extension points for:
{% top %}
-Where to go next?
+接下来？
-----------------
-* [Concepts & Common API]({{ site.baseurl }}/dev/table/common.html): Shared concepts and APIs of the Table API and SQL.
-* [Data Types]({{ site.baseurl }}/dev/table/types.html): Lists pre-defined data types and their properties.
-* [Streaming Concepts]({{ site.baseurl }}/dev/table/streaming): Streaming-specific documentation for the Table API or SQL such as configuration of time attributes and handling of updating results.
-* [Connect to External Systems]({{ site.baseurl }}/dev/table/functions.html): Available connectors and formats for reading and writing data to external systems.
-* [Table API]({{ site.baseurl }}/dev/table/tableApi.html): Supported operations and API for the Table API.
-* [SQL]({{ site.baseurl }}/dev/table/sql.html): Supported operations and syntax for SQL.
-* [Built-in Functions]({{ site.baseurl }}/dev/table/functions.html): Supported functions in Table API and SQL.
-* [SQL Client]({{ site.baseurl }}/dev/table/sqlClient.html): Play around with Flink SQL and submit a table program to a cluster without programming knowledge.
+* [公共概念和 API]({{ site.baseurl }}/zh/dev/table/common.html): Table API 和 SQL 公共概念以及 API。
+* [数据类型]({{ site.baseurl }}/zh/dev/table/types.html): 内置数据类型以及它们的属性
+* [流式概念]({{ site.baseurl }}/zh/dev/table/streaming): Table API 和 SQL 中流式相关的文档，比如配置时间属性和如何处理更新结果。
+* [连接外部系统]({{ site.baseurl }}/zh/dev/table/connect.html): 读写外部系统的连接器和格式。
+* [Table API]({{ site.baseurl }}/zh/dev/table/tableApi.html): Table API 支持的操作。
+* [SQL]({{ site.baseurl }}/zh/dev/table/sql/index.html): SQL 支持的操作和语法。
+* [内置函数]({{ site.baseurl }}/zh/dev/table/functions/systemFunctions.html): Table API 和 SQL 中的内置函数。
+* [SQL Client]({{ site.baseurl }}/zh/dev/table/sqlClient.html): 不用编写代码就可以尝试 Flink SQL，可以直接提交 SQL 任务到集群上。
{% top %}
diff --git a/docs/dev/table/modules.md b/docs/dev/table/modules.md
new file mode 100644
index 0000000000000000000000000000000000000000..ec39a08c05fa9775f4c5db9c38ad5e1e573df4c6
--- /dev/null
+++ b/docs/dev/table/modules.md
@@ -0,0 +1,126 @@
+---
+title: "Modules"
+is_beta: true
+nav-parent_id: tableapi
+nav-pos: 70
+---
+
+
+Modules allow users to extend Flink's built-in objects, such as defining functions that behave like Flink
+built-in functions. They are pluggable, and while Flink provides a few pre-built modules, users can write
+their own.
+
+For example, users can define their own geo functions and plug them into Flink as built-in functions to be used in
+Flink SQL and Table APIs. Another example is users can load an out-of-shelf Hive module to use Hive built-in
+functions as Flink built-in functions.
+
+* This will be replaced by the TOC
+{:toc}
+
+## Module Types
+
+### CoreModule
+
+`CoreModule` contains all of Flink's system (built-in) functions and is loaded by default.
+
+### HiveModule
+
+The `HiveModule` provides Hive built-in functions as Flink's system functions to SQL and Table API users.
+Flink's [Hive documentation]({{ site.baseurl }}/dev/table/hive/hive_functions.html) provides full details on setting up the module.
+
+### User-Defined Module
+
+Users can develop custom modules by implementing the `Module` interface.
+To use custom modules in SQL CLI, users should develop both a module and its corresponding module factory by implementing
+the `ModuleFactory` interface.
+
+A module factory defines a set of properties for configuring the module when the SQL CLI bootstraps.
+Properties are passed to a discovery service where the service tries to match the properties to
+ a `ModuleFactory` and instantiate a corresponding module instance.
+
+
+## Namespace and Resolution Order
+
+Objects provided by modules are considered part of Flink's system (built-in) objects; thus, they don't have any namespaces.
+
+When there are two objects of the same name residing in two modules, Flink always resolves the object reference to the one in the 1st loaded module.
+
+## Module API
+
+### Loading and unloading a Module
+
+Users can load and unload modules in an existing Flink session.
+
+
diff --git a/docs/dev/table/modules.zh.md b/docs/dev/table/modules.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..8367865bed9ae11f7e08ecff4844480569a493eb
--- /dev/null
+++ b/docs/dev/table/modules.zh.md
@@ -0,0 +1,126 @@
+---
+title: "模块"
+is_beta: true
+nav-parent_id: tableapi
+nav-pos: 70
+---
+
+
+Modules allow users to extend Flink's built-in objects, such as defining functions that behave like Flink
+built-in functions. They are pluggable, and while Flink provides a few pre-built modules, users can write
+their own.
+
+For example, users can define their own geo functions and plug them into Flink as built-in functions to be used in
+Flink SQL and Table APIs. Another example is users can load an out-of-shelf Hive module to use Hive built-in
+functions as Flink built-in functions.
+
+* This will be replaced by the TOC
+{:toc}
+
+## Module Types
+
+### CoreModule
+
+`CoreModule` contains all of Flink's system (built-in) functions and is loaded by default.
+
+### HiveModule
+
+The `HiveModule` provides Hive built-in functions as Flink's system functions to SQL and Table API users.
+Flink's [Hive documentation]({{ site.baseurl }}/dev/table/hive/hive_functions.html) provides full details on setting up the module.
+
+### User-Defined Module
+
+Users can develop custom modules by implementing the `Module` interface.
+To use custom modules in SQL CLI, users should develop both a module and its corresponding module factory by implementing
+the `ModuleFactory` interface.
+
+A module factory defines a set of properties for configuring the module when the SQL CLI bootstraps.
+Properties are passed to a discovery service where the service tries to match the properties to
+ a `ModuleFactory` and instantiate a corresponding module instance.
+
+
+## Namespace and Resolution Order
+
+Objects provided by modules are considered part of Flink's system (built-in) objects; thus, they don't have any namespaces.
+
+When there are two objects of the same name residing in two modules, Flink always resolves the object reference to the one in the 1st loaded module.
+
+## Module API
+
+### Loading and unloading a Module
+
+Users can load and unload modules in an existing Flink session.
+
+
diff --git a/docs/dev/table/sourceSinks.md b/docs/dev/table/sourceSinks.md
index 8da8046ee42ff28f52cfef46394c3ba91ed635ad..dcd6829d5b6f8d7070ba5f8f96ad01e55f85dfd8 100644
--- a/docs/dev/table/sourceSinks.md
+++ b/docs/dev/table/sourceSinks.md
@@ -1,7 +1,7 @@
---
title: "User-defined Sources & Sinks"
nav-parent_id: tableapi
-nav-pos: 40
+nav-pos: 130
---
-A `TableSource` provides access to data which is stored in external systems (database, key-value store, message queue) or files. After a [TableSource is registered in a TableEnvironment](common.html#register-a-tablesource) it can be accessed by [Table API](tableApi.html) or [SQL](sql.html) queries.
+A `TableSource` provides access to data which is stored in external systems (database, key-value store, message queue) or files. After a [TableSource is registered in a TableEnvironment](common.html#register-a-tablesource) it can be accessed by [Table API](tableApi.html) or [SQL]({{ site.baseurl }}/dev/table/sql/queries.html) queries.
A `TableSink` [emits a Table](common.html#emit-a-table) to an external storage system, such as a database, key-value store, message queue, or file system (in different encodings, e.g., CSV, Parquet, or ORC).
@@ -71,7 +71,7 @@ TableSource[T] {
-* `getTableSchema()`: Returns the schema of the table, i.e., the names and types of the fields of the table. The field types are defined using Flink's `TypeInformation` (see [Table API types](tableApi.html#data-types) and [SQL types](sql.html#data-types)).
+* `getTableSchema()`: Returns the schema of the produced table, i.e., the names and types of the fields of the table. The field types are defined using Flink's `DataType` (see [Table API types]({{ site.baseurl }}/dev/table/types.html) and [SQL types]({{ site.baseurl }}/dev/table/sql/index.html#data-types)). Note that the returned `TableSchema` shouldn't contain computed columns to reflect the schema of the physical `TableSource`.
* `getReturnType()`: Returns the physical type of the `DataStream` (`StreamTableSource`) or `DataSet` (`BatchTableSource`) and the records that are produced by the `TableSource`.
@@ -139,7 +139,7 @@ StreamTableSource[T] extends TableSource[T] {
### Defining a TableSource with Time Attributes
-Time-based operations of streaming [Table API](tableApi.html#group-windows) and [SQL](sql.html#group-windows) queries, such as windowed aggregations or joins, require explicitly specified [time attributes](streaming/time_attributes.html).
+Time-based operations of streaming [Table API](tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#group-windows) queries, such as windowed aggregations or joins, require explicitly specified [time attributes](streaming/time_attributes.html).
A `TableSource` defines a time attribute as a field of type `Types.SQL_TIMESTAMP` in its table schema. In contrast to all regular fields in the schema, a time attribute must not be matched to a physical field in the return type of the table source. Instead, a `TableSource` defines a time attribute by implementing a certain interface.
@@ -367,7 +367,7 @@ LookupableTableSource[T] extends TableSource[T] {
* `getLookupFunction(lookupkeys)`: Returns a `TableFunction` which used to lookup the matched row(s) via lookup keys. The lookupkeys are the field names of `LookupableTableSource` in the join equal conditions. The eval method parameters of the returned `TableFunction`'s should be in the order which `lookupkeys` defined. It is recommended to define the parameters in varargs (e.g. `eval(Object... lookupkeys)` to match all the cases). The return type of the `TableFunction` must be identical to the return type defined by the `TableSource.getReturnType()` method.
-* `getAsyncLookupFunction(lookupkeys)`: Optional. Similar to `getLookupFunction`, but the `AsyncLookupFunction` lookups the matched row(s) asynchronously. The underlying of `AsyncLookupFunction` will be called via [Async I/O](/dev/stream/operators/asyncio.html). The first argument of the eval method of the returned `AsyncTableFunction` should be defined as `java.util.concurrent.CompletableFuture` to collect results asynchronously (e.g. `eval(CompletableFuture> result, Object... lookupkeys)`). The implementation of this method can throw an exception if the TableSource doesn't support asynchronously lookup.
+* `getAsyncLookupFunction(lookupkeys)`: Optional. Similar to `getLookupFunction`, but the `AsyncLookupFunction` lookups the matched row(s) asynchronously. The underlying of `AsyncLookupFunction` will be called via [Async I/O]({{ site.baseurl }}/dev/stream/operators/asyncio.html). The first argument of the eval method of the returned `AsyncTableFunction` should be defined as `java.util.concurrent.CompletableFuture` to collect results asynchronously (e.g. `eval(CompletableFuture> result, Object... lookupkeys)`). The implementation of this method can throw an exception if the TableSource doesn't support asynchronously lookup.
* `isAsyncEnabled()`: Returns true if async lookup is enabled. It requires `getAsyncLookupFunction(lookupkeys)` is implemented if `isAsyncEnabled` returns true.
{% top %}
@@ -411,7 +411,7 @@ TableSink[T] {
-The `TableSink#configure` method is called to pass the schema of the Table (field names and types) to emit to the `TableSink`. The method must return a new instance of the TableSink which is configured to emit the provided Table schema.
+The `TableSink#configure` method is called to pass the schema of the Table (field names and types) to emit to the `TableSink`. The method must return a new instance of the TableSink which is configured to emit the provided Table schema. Note that the provided `TableSchema` shouldn't contain computed columns to reflect the schema of the physical `TableSink`.
### BatchTableSink
@@ -714,10 +714,11 @@ connector.debug=true
For a type-safe, programmatic approach with explanatory Scaladoc/Javadoc, the Table & SQL API offers descriptors in `org.apache.flink.table.descriptors` that translate into string-based properties. See the [built-in descriptors](connect.html) for sources, sinks, and formats as a reference.
-A connector for `MySystem` in our example can extend `ConnectorDescriptor` as shown below:
-
+
+A custom descriptor can be defined by extending the `ConnectorDescriptor` class.
+
{% highlight java %}
import org.apache.flink.table.descriptors.ConnectorDescriptor;
import java.util.HashMap;
@@ -743,9 +744,25 @@ public class MySystemConnector extends ConnectorDescriptor {
}
}
{% endhighlight %}
+
+The descriptor can then be used to create a table with the table environment.
+
+{% highlight java %}
+StreamTableEnvironment tableEnv = // ...
+
+tableEnv
+ .connect(new MySystemConnector(true))
+ .withSchema(...)
+ .inAppendMode()
+ .createTemporaryTable("MySystemTable");
+{% endhighlight %}
+
+
+A custom descriptor can be defined by extending the `ConnectorDescriptor` class.
+
{% highlight scala %}
import org.apache.flink.table.descriptors.ConnectorDescriptor
import java.util.HashMap
@@ -763,33 +780,38 @@ class MySystemConnector(isDebug: Boolean) extends ConnectorDescriptor("my-system
}
}
{% endhighlight %}
-
-
-The descriptor can then be used in the API as follows:
+The descriptor can then be used to create a table with the table environment.
-
-tableEnv
- .connect(new MySystemConnector(isDebug = true))
- .inAppendMode()
- .registerTableSource("MySystemTable")
+You can use a Java `TableFactory` from Python using the `CustomConnectorDescriptor`.
+
+{% highlight python %}
+s_env = StreamExecutionEnvironment.get_execution_environment()
+st_env = StreamTableEnvironment.create(s_env)
+
+custom_connector = CustomConnectorDescriptor('my-system', 1, False)
+st_env\
+ .connect(custom_connector.property("connector.debug", "true")) \
+ .with_schema(...) \
+ .in_append_mode()\
+ .create_temporary_table("MySystemTable")
{% endhighlight %}
+
{% top %}
diff --git a/docs/dev/table/sourceSinks.zh.md b/docs/dev/table/sourceSinks.zh.md
index 82398a44d873134f21eba44c318c9f67391e514d..fab6930dbee41b4bc2dbcbb98e8a0814519d5d1f 100644
--- a/docs/dev/table/sourceSinks.zh.md
+++ b/docs/dev/table/sourceSinks.zh.md
@@ -1,7 +1,7 @@
---
title: "User-defined Sources & Sinks"
nav-parent_id: tableapi
-nav-pos: 40
+nav-pos: 130
---
-A `TableSource` provides access to data which is stored in external systems (database, key-value store, message queue) or files. After a [TableSource is registered in a TableEnvironment](common.html#register-a-tablesource) it can be accessed by [Table API](tableApi.html) or [SQL](sql.html) queries.
+A `TableSource` provides access to data which is stored in external systems (database, key-value store, message queue) or files. After a [TableSource is registered in a TableEnvironment](common.html#register-a-tablesource) it can be accessed by [Table API](tableApi.html) or [SQL]({{ site.baseurl }}/dev/table/sql/queries.html) queries.
A `TableSink` [emits a Table](common.html#emit-a-table) to an external storage system, such as a database, key-value store, message queue, or file system (in different encodings, e.g., CSV, Parquet, or ORC).
@@ -36,9 +36,9 @@ Have a look at the [common concepts and API](common.html) page for details how t
Define a TableSource
--------------------
-A `TableSource` is a generic interface that gives Table API and SQL queries access to data stored in an external system. It provides the schema of the table and the records that are mapped to rows with the table's schema. Depending on whether the `TableSource` is used in a streaming or batch query, the records are produced as a `DataSet` or `DataStream`.
+A `TableSource` is a generic interface that gives Table API and SQL queries access to data stored in an external system. It provides the schema of the table and the records that are mapped to rows with the table's schema. Depending on whether the `TableSource` is used in a streaming or batch query, the records are produced as a `DataSet` or `DataStream`.
-If a `TableSource` is used in a streaming query it must implement the `StreamTableSource` interface, if it is used in a batch query it must implement the `BatchTableSource` interface. A `TableSource` can also implement both interfaces and be used in streaming and batch queries.
+If a `TableSource` is used in a streaming query it must implement the `StreamTableSource` interface, if it is used in a batch query it must implement the `BatchTableSource` interface. A `TableSource` can also implement both interfaces and be used in streaming and batch queries.
`StreamTableSource` and `BatchTableSource` extend the base interface `TableSource` that defines the following methods:
@@ -71,7 +71,7 @@ TableSource[T] {
-* `getTableSchema()`: Returns the schema of the table, i.e., the names and types of the fields of the table. The field types are defined using Flink's `TypeInformation` (see [Table API types](tableApi.html#data-types) and [SQL types](sql.html#data-types)).
+* `getTableSchema()`: Returns the schema of the produced table, i.e., the names and types of the fields of the table. The field types are defined using Flink's `DataType` (see [Table API types]({{ site.baseurl }}/dev/table/types.html) and [SQL types]({{ site.baseurl }}/dev/table/sql/index.html#data-types)). Note that the returned `TableSchema` shouldn't contain computed columns to reflect the schema of the physical `TableSource`.
* `getReturnType()`: Returns the physical type of the `DataStream` (`StreamTableSource`) or `DataSet` (`BatchTableSource`) and the records that are produced by the `TableSource`.
@@ -139,7 +139,7 @@ StreamTableSource[T] extends TableSource[T] {
### Defining a TableSource with Time Attributes
-Time-based operations of streaming [Table API](tableApi.html#group-windows) and [SQL](sql.html#group-windows) queries, such as windowed aggregations or joins, require explicitly specified [time attributes](streaming/time_attributes.html).
+Time-based operations of streaming [Table API](tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#group-windows) queries, such as windowed aggregations or joins, require explicitly specified [time attributes](streaming/time_attributes.html).
A `TableSource` defines a time attribute as a field of type `Types.SQL_TIMESTAMP` in its table schema. In contrast to all regular fields in the schema, a time attribute must not be matched to a physical field in the return type of the table source. Instead, a `TableSource` defines a time attribute by implementing a certain interface.
@@ -367,7 +367,7 @@ LookupableTableSource[T] extends TableSource[T] {
* `getLookupFunction(lookupkeys)`: Returns a `TableFunction` which used to lookup the matched row(s) via lookup keys. The lookupkeys are the field names of `LookupableTableSource` in the join equal conditions. The eval method parameters of the returned `TableFunction`'s should be in the order which `lookupkeys` defined. It is recommended to define the parameters in varargs (e.g. `eval(Object... lookupkeys)` to match all the cases). The return type of the `TableFunction` must be identical to the return type defined by the `TableSource.getReturnType()` method.
-* `getAsyncLookupFunction(lookupkeys)`: Optional. Similar to `getLookupFunction`, but the `AsyncLookupFunction` lookups the matched row(s) asynchronously. The underlying of `AsyncLookupFunction` will be called via [Async I/O](/dev/stream/operators/asyncio.html). The first argument of the eval method of the returned `AsyncTableFunction` should be defined as `java.util.concurrent.CompletableFuture` to collect results asynchronously (e.g. `eval(CompletableFuture> result, Object... lookupkeys)`). The implementation of this method can throw an exception if the TableSource doesn't support asynchronously lookup.
+* `getAsyncLookupFunction(lookupkeys)`: Optional. Similar to `getLookupFunction`, but the `AsyncLookupFunction` lookups the matched row(s) asynchronously. The underlying of `AsyncLookupFunction` will be called via [Async I/O]({{ site.baseurl }}/dev/stream/operators/asyncio.html). The first argument of the eval method of the returned `AsyncTableFunction` should be defined as `java.util.concurrent.CompletableFuture` to collect results asynchronously (e.g. `eval(CompletableFuture> result, Object... lookupkeys)`). The implementation of this method can throw an exception if the TableSource doesn't support asynchronously lookup.
* `isAsyncEnabled()`: Returns true if async lookup is enabled. It requires `getAsyncLookupFunction(lookupkeys)` is implemented if `isAsyncEnabled` returns true.
{% top %}
@@ -411,7 +411,7 @@ TableSink[T] {
-The `TableSink#configure` method is called to pass the schema of the Table (field names and types) to emit to the `TableSink`. The method must return a new instance of the TableSink which is configured to emit the provided Table schema.
+The `TableSink#configure` method is called to pass the schema of the Table (field names and types) to emit to the `TableSink`. The method must return a new instance of the TableSink which is configured to emit the provided Table schema. Note that the provided `TableSchema` shouldn't contain computed columns to reflect the schema of the physical `TableSink`.
### BatchTableSink
@@ -714,10 +714,11 @@ connector.debug=true
For a type-safe, programmatic approach with explanatory Scaladoc/Javadoc, the Table & SQL API offers descriptors in `org.apache.flink.table.descriptors` that translate into string-based properties. See the [built-in descriptors](connect.html) for sources, sinks, and formats as a reference.
-A connector for `MySystem` in our example can extend `ConnectorDescriptor` as shown below:
-
+
+A custom descriptor can be defined by extending the `ConnectorDescriptor` class.
+
{% highlight java %}
import org.apache.flink.table.descriptors.ConnectorDescriptor;
import java.util.HashMap;
@@ -743,9 +744,25 @@ public class MySystemConnector extends ConnectorDescriptor {
}
}
{% endhighlight %}
+
+The descriptor can then be used to create a table with the table environment.
+
+{% highlight java %}
+StreamTableEnvironment tableEnv = // ...
+
+tableEnv
+ .connect(new MySystemConnector(true))
+ .withSchema(...)
+ .inAppendMode()
+ .createTemporaryTable("MySystemTable");
+{% endhighlight %}
+
+
+A custom descriptor can be defined by extending the `ConnectorDescriptor` class.
+
{% highlight scala %}
import org.apache.flink.table.descriptors.ConnectorDescriptor
import java.util.HashMap
@@ -755,7 +772,7 @@ import java.util.Map
* Connector to MySystem with debug mode.
*/
class MySystemConnector(isDebug: Boolean) extends ConnectorDescriptor("my-system", 1, false) {
-
+
override protected def toConnectorProperties(): Map[String, String] = {
val properties = new HashMap[String, String]
properties.put("connector.debug", isDebug.toString)
@@ -763,33 +780,38 @@ class MySystemConnector(isDebug: Boolean) extends ConnectorDescriptor("my-system
}
}
{% endhighlight %}
-
-
-The descriptor can then be used in the API as follows:
+The descriptor can then be used to create a table with the table environment.
-
-tableEnv
- .connect(new MySystemConnector(isDebug = true))
- .inAppendMode()
- .registerTableSource("MySystemTable")
+You can use a Java `TableFactory` from Python using the `CustomConnectorDescriptor`.
+
+{% highlight python %}
+s_env = StreamExecutionEnvironment.get_execution_environment()
+st_env = StreamTableEnvironment.create(s_env)
+
+custom_connector = CustomConnectorDescriptor('my-system', 1, False)
+st_env\
+ .connect(custom_connector.property("connector.debug", "true")) \
+ .with_schema(...) \
+ .in_append_mode()\
+ .create_temporary_table("MySystemTable")
{% endhighlight %}
+
{% top %}
diff --git a/docs/dev/table/sql/alter.md b/docs/dev/table/sql/alter.md
new file mode 100644
index 0000000000000000000000000000000000000000..4cba0828e1d345460e0df8570f314b93d2884556
--- /dev/null
+++ b/docs/dev/table/sql/alter.md
@@ -0,0 +1,165 @@
+---
+title: "ALTER Statements"
+nav-parent_id: sql
+nav-pos: 4
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+ALTER statements are used to modified a registered table/view/function definition in the [Catalog]({{ site.baseurl }}/dev/table/catalogs.html).
+
+Flink SQL supports the following ALTER statements for now:
+
+- ALTER TABLE
+- ALTER DATABASE
+- ALTER FUNCTION
+
+## Run an ALTER statement
+
+ALTER statements can be executed with the `sqlUpdate()` method of the `TableEnvironment`, or executed in [SQL CLI]({{ site.baseurl }}/dev/table/sqlClient.html). The `sqlUpdate()` method returns nothing for a successful ALTER operation, otherwise will throw an exception.
+
+The following examples show how to run an ALTER statement in `TableEnvironment` and in SQL CLI.
+
+
+
+{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance()...
+TableEnvironment tableEnv = TableEnvironment.create(settings);
+
+// register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+
+// a string array: ["Orders"]
+String[] tables = tableEnv.listTable();
+
+// rename "Orders" to "NewOrders"
+tableEnv.sqlUpdate("ALTER TABLE Orders RENAME TO NewOrders;");
+
+// a string array: ["NewOrders"]
+String[] tables = tableEnv.listTable();
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+val settings = EnvironmentSettings.newInstance()...
+val tableEnv = TableEnvironment.create(settings)
+
+// register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+
+// a string array: ["Orders"]
+val tables = tableEnv.listTable()
+
+// rename "Orders" to "NewOrders"
+tableEnv.sqlUpdate("ALTER TABLE Orders RENAME TO NewOrders;")
+
+// a string array: ["NewOrders"]
+val tables = tableEnv.listTable()
+{% endhighlight %}
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> SHOW TABLES;
+Orders
+
+Flink SQL> ALTER TABLE Orders RENAME TO NewOrders;
+[INFO] Table has been removed.
+
+Flink SQL> SHOW TABLES;
+NewOrders
+{% endhighlight %}
+
+
+
+## ALTER TABLE
+
+* Rename Table
+
+{% highlight sql %}
+ALTER TABLE [catalog_name.][db_name.]table_name RENAME TO new_table_name
+{% endhighlight %}
+
+Rename the given table name to another new table name.
+
+* Set or Alter Table Properties
+
+{% highlight sql %}
+ALTER TABLE [catalog_name.][db_name.]table_name SET (key1=val1, key2=val2, ...)
+{% endhighlight %}
+
+Set one or more properties in the specified table. If a particular property is already set in the table, override the old value with the new one.
+
+## ALTER DATABASE
+
+{% highlight sql %}
+ALTER DATABASE [catalog_name.]db_name SET (key1=val1, key2=val2, ...)
+{% endhighlight %}
+
+Set one or more properties in the specified database. If a particular property is already set in the database, override the old value with the new one.
+
+## ALTER FUNCTION
+
+{% highlight sql%}
+ALTER [TEMPORARY|TEMPORARY SYSTEM] FUNCTION
+ [IF EXISTS] [catalog_name.][db_name.]function_name
+ AS identifier [LANGUAGE JAVA|SCALA|
+{% endhighlight %}
+
+Alter a catalog function with the new identifier which is full classpath for JAVA/SCALA and optional language tag. If a function doesn't exist in the catalog, an exception is thrown.
+
+**TEMPORARY**
+
+Alter temporary catalog function that has catalog and database namespaces and overrides catalog functions.
+
+**TEMPORARY SYSTEM**
+
+Alter temporary system function that has no namespace and overrides built-in functions
+
+**IF EXISTS**
+
+If the function doesn't exist, nothing happens.
+
+**LANGUAGE JAVA\|SCALA**
+
+Language tag to instruct flink runtime how to execute the function. Currently only JAVA and SCALA are supported, the default language for a function is JAVA.
+
diff --git a/docs/dev/table/sql/alter.zh.md b/docs/dev/table/sql/alter.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..def2862c2f03c1fba7ed164caa662a8d365bdd70
--- /dev/null
+++ b/docs/dev/table/sql/alter.zh.md
@@ -0,0 +1,164 @@
+---
+title: "ALTER 语句"
+nav-parent_id: sql
+nav-pos: 4
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+ALTER 语句用于修改一个已经在 [Catalog]({{ site.baseurl }}/zh/dev/table/catalogs.html) 中注册的表、视图或函数定义。
+
+Flink SQL 目前支持以下 ALTER 语句：
+
+- ALTER TABLE
+- ALTER DATABASE
+- ALTER FUNCTION
+
+## 执行 ALTER 语句
+
+可以使用 `TableEnvironment` 中的 `sqlUpdate()` 方法执行 ALTER 语句，也可以在 [SQL CLI]({{ site.baseurl }}/zh/dev/table/sqlClient.html) 中执行 ALTER 语句。 若 ALTER 操作执行成功，`sqlUpdate()` 方法不返回任何内容，否则会抛出异常。
+
+以下的例子展示了如何在 `TableEnvironment` 和 SQL CLI 中执行一个 ALTER 语句。
+
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> SHOW TABLES;
+Orders
+
+Flink SQL> ALTER TABLE Orders RENAME TO NewOrders;
+[INFO] Table has been removed.
+
+Flink SQL> SHOW TABLES;
+NewOrders
+{% endhighlight %}
+
+
+
+## ALTER TABLE
+
+* 重命名表
+
+{% highlight sql %}
+ALTER TABLE [catalog_name.][db_name.]table_name RENAME TO new_table_name
+{% endhighlight %}
+
+把原有的表名更改为新的表名。
+
+* 设置或修改表属性
+
+{% highlight sql %}
+ALTER TABLE [catalog_name.][db_name.]table_name SET (key1=val1, key2=val2, ...)
+{% endhighlight %}
+
+为指定的表设置一个或多个属性。若个别属性已经存在于表中，则使用新的值覆盖旧的值。
+
+## ALTER DATABASE
+
+{% highlight sql %}
+ALTER DATABASE [catalog_name.]db_name SET (key1=val1, key2=val2, ...)
+{% endhighlight %}
+
+在数据库中设置一个或多个属性。若个别属性已经在数据库中设定，将会使用新值覆盖旧值。
+
+## ALTER FUNCTION
+
+{% highlight sql%}
+ALTER [TEMPORARY|TEMPORARY SYSTEM] FUNCTION
+ [IF EXISTS] [catalog_name.][db_name.]function_name
+ AS identifier [LANGUAGE JAVA|SCALA|
+{% endhighlight %}
+
+修改一个有 catalog 和数据库命名空间的 catalog function ，其需要指定 JAVA / SCALA 或其他 language tag 完整的 classpath。若函数不存在，删除会抛出异常。
+
+**TEMPORARY**
+
+修改一个有 catalog 和数据库命名空间的临时 catalog function ，并覆盖原有的 catalog function 。
+
+**TEMPORARY SYSTEM**
+
+修改一个没有数据库命名空间的临时系统 catalog function ，并覆盖系统内置的函数。
+
+**IF EXISTS**
+
+若函数不存在，则不进行任何操作。
+
+**LANGUAGE JAVA\|SCALA**
+
+Language tag 用于指定 Flink runtime 如何执行这个函数。目前，只支持 JAVA 和 SCALA，且函数的默认语言为 JAVA。
\ No newline at end of file
diff --git a/docs/dev/table/sql/create.md b/docs/dev/table/sql/create.md
new file mode 100644
index 0000000000000000000000000000000000000000..6e41eac277e517eb44face8977fc9b392e9663e3
--- /dev/null
+++ b/docs/dev/table/sql/create.md
@@ -0,0 +1,257 @@
+---
+title: "CREATE Statements"
+nav-parent_id: sql
+nav-pos: 2
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+CREATE statements are used to register a table/view/function into current or specified [Catalog]({{ site.baseurl }}/dev/table/catalogs.html). A registered table/view/function can be used in SQL queries.
+
+Flink SQL supports the following CREATE statements for now:
+
+- CREATE TABLE
+- CREATE DATABASE
+- CREATE FUNCTION
+
+## Run a CREATE statement
+
+CREATE statements can be executed with the `sqlUpdate()` method of the `TableEnvironment`, or executed in [SQL CLI]({{ site.baseurl }}/dev/table/sqlClient.html). The `sqlUpdate()` method returns nothing for a successful CREATE operation, otherwise will throw an exception.
+
+The following examples show how to run a CREATE statement in `TableEnvironment` and in SQL CLI.
+
+
+
+{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance()...
+TableEnvironment tableEnv = TableEnvironment.create(settings);
+
+// SQL query with a registered table
+// register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+// run a SQL query on the Table and retrieve the result as a new Table
+Table result = tableEnv.sqlQuery(
+ "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+
+// SQL update with a registered table
+// register a TableSink
+tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)");
+// run a SQL update query on the Table and emit the result to the TableSink
+tableEnv.sqlUpdate(
+ "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+val settings = EnvironmentSettings.newInstance()...
+val tableEnv = TableEnvironment.create(settings)
+
+// SQL query with a registered table
+// register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+// run a SQL query on the Table and retrieve the result as a new Table
+val result = tableEnv.sqlQuery(
+ "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+
+// SQL update with a registered table
+// register a TableSink
+tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH ('connector.path'='/path/to/file' ...)");
+// run a SQL update query on the Table and emit the result to the TableSink
+tableEnv.sqlUpdate(
+ "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight python %}
+settings = EnvironmentSettings.newInstance()...
+table_env = TableEnvironment.create(settings)
+
+# SQL query with a registered table
+# register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+# run a SQL query on the Table and retrieve the result as a new Table
+result = tableEnv.sqlQuery(
+ "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+
+# SQL update with a registered table
+# register a TableSink
+table_env.sql_update("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)")
+# run a SQL update query on the Table and emit the result to the TableSink
+table_env \
+ .sql_update("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> CREATE TABLE RubberOrders (product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%';
+[INFO] Submitting SQL update statement to the cluster...
+{% endhighlight %}
+
+
+
+{% top %}
+
+## CREATE TABLE
+
+{% highlight sql %}
+CREATE TABLE [catalog_name.][db_name.]table_name
+ (
+ { | }[ , ...n]
+ [ ]
+ )
+ [COMMENT table_comment]
+ [PARTITIONED BY (partition_column_name1, partition_column_name2, ...)]
+ WITH (key1=val1, key2=val2, ...)
+
+:
+ column_name column_type [COMMENT column_comment]
+
+:
+ column_name AS computed_column_expression [COMMENT column_comment]
+
+:
+ WATERMARK FOR rowtime_column_name AS watermark_strategy_expression
+
+{% endhighlight %}
+
+Creates a table with the given name. If a table with the same name already exists in the catalog, an exception is thrown.
+
+**COMPUTED COLUMN**
+
+A computed column is a virtual column that is generated using the syntax "`column_name AS computed_column_expression`". It is generated from a non-query expression that uses other columns in the same table and is not physically stored within the table. For example, a computed column could be defined as `cost AS price * quantity`. The expression may contain any combination of physical column, constant, function, or variable. The expression cannot contain a subquery.
+
+Computed columns are commonly used in Flink for defining [time attributes]({{ site.baseurl}}/dev/table/streaming/time_attributes.html) in CREATE TABLE statements.
+A [processing time attribute]({{ site.baseurl}}/dev/table/streaming/time_attributes.html#processing-time) can be defined easily via `proc AS PROCTIME()` using the system `PROCTIME()` function.
+On the other hand, computed column can be used to derive event time column because an event time column may need to be derived from existing fields, e.g. the original field is not `TIMESTAMP(3)` type or is nested in a JSON string.
+
+Notes:
+
+- A computed column defined on a source table is computed after reading from the source, it can be used in the following SELECT query statements.
+- A computed column cannot be the target of an INSERT statement. In INSERT statements, the schema of SELECT clause should match the schema of the target table without computed columns.
+
+**WATERMARK**
+
+The `WATERMARK` defines the event time attributes of a table and takes the form `WATERMARK FOR rowtime_column_name AS watermark_strategy_expression`.
+
+The `rowtime_column_name` defines an existing column that is marked as the event time attribute of the table. The column must be of type `TIMESTAMP(3)` and be a top-level column in the schema. It may be a computed column.
+
+The `watermark_strategy_expression` defines the watermark generation strategy. It allows arbitrary non-query expression, including computed columns, to calculate the watermark. The expression return type must be TIMESTAMP(3), which represents the timestamp since the Epoch.
+The returned watermark will be emitted only if it is non-null and its value is larger than the previously emitted local watermark (to preserve the contract of ascending watermarks). The watermark generation expression is evaluated by the framework for every record.
+The framework will periodically emit the largest generated watermark. If the current watermark is still identical to the previous one, or is null, or the value of the returned watermark is smaller than that of the last emitted one, then no new watermark will be emitted.
+Watermark is emitted in an interval defined by [`pipeline.auto-watermark-interval`]({{ site.baseurl }}/ops/config.html#pipeline-auto-watermark-interval) configuration.
+If watermark interval is `0ms`, the generated watermarks will be emitted per-record if it is not null and greater than the last emitted one.
+
+When using event time semantics, tables must contain an event time attribute and watermarking strategy.
+
+Flink provides several commonly used watermark strategies.
+
+- Strictly ascending timestamps: `WATERMARK FOR rowtime_column AS rowtime_column`.
+
+ Emits a watermark of the maximum observed timestamp so far. Rows that have a timestamp smaller to the max timestamp are not late.
+
+- Ascending timestamps: `WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL '0.001' SECOND`.
+
+ Emits a watermark of the maximum observed timestamp so far minus 1. Rows that have a timestamp equal and smaller to the max timestamp are not late.
+
+- Bounded out of orderness timestamps: `WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL 'string' timeUnit`.
+
+ Emits watermarks, which are the maximum observed timestamp minus the specified delay, e.g., `WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL '5' SECOND` is a 5 seconds delayed watermark strategy.
+
+{% highlight sql %}
+CREATE TABLE Orders (
+ user BIGINT,
+ product STRING,
+ order_time TIMESTAMP(3),
+ WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND
+) WITH ( . . . );
+{% endhighlight %}
+
+**PARTITIONED BY**
+
+Partition the created table by the specified columns. A directory is created for each partition if this table is used as a filesystem sink.
+
+**WITH OPTIONS**
+
+Table properties used to create a table source/sink. The properties are usually used to find and create the underlying connector.
+
+The key and value of expression `key1=val1` should both be string literal. See details in [Connect to External Systems]({{ site.baseurl }}/dev/table/connect.html) for all the supported table properties of different connectors.
+
+**Notes:** The table name can be of three formats: 1. `catalog_name.db_name.table_name` 2. `db_name.table_name` 3. `table_name`. For `catalog_name.db_name.table_name`, the table would be registered into metastore with catalog named "catalog_name" and database named "db_name"; for `db_name.table_name`, the table would be registered into the current catalog of the execution table environment and database named "db_name"; for `table_name`, the table would be registered into the current catalog and database of the execution table environment.
+
+**Notes:** The table registered with `CREATE TABLE` statement can be used as both table source and table sink, we can not decide if it is used as a source or sink until it is referenced in the DMLs.
+
+{% top %}
+
+## CREATE DATABASE
+
+{% highlight sql %}
+CREATE DATABASE [IF NOT EXISTS] [catalog_name.]db_name
+ [COMMENT database_comment]
+ WITH (key1=val1, key2=val2, ...)
+{% endhighlight %}
+
+Create a database with the given database properties. If a database with the same name already exists in the catalog, an exception is thrown.
+
+**IF NOT EXISTS**
+
+If the database already exists, nothing happens.
+
+**WITH OPTIONS**
+
+Database properties used to store extra information related to this database.
+The key and value of expression `key1=val1` should both be string literal.
+
+{% top %}
+
+## CREATE FUNCTION
+{% highlight sql%}
+CREATE [TEMPORARY|TEMPORARY SYSTEM] FUNCTION
+ [IF NOT EXISTS] [catalog_name.][db_name.]function_name
+ AS identifier [LANGUAGE JAVA|SCALA]
+{% endhighlight %}
+
+Create a catalog function that has catalog and database namespaces with the identifier which is full classpath for JAVA/SCALA and optional language tag. If a function with the same name already exists in the catalog, an exception is thrown.
+
+**TEMPORARY**
+
+Create temporary catalog function that has catalog and database namespaces and overrides catalog functions.
+
+**TEMPORARY SYSTEM**
+
+Create temporary system function that has no namespace and overrides built-in functions
+
+**IF NOT EXISTS**
+
+If the function already exists, nothing happens.
+
+**LANGUAGE JAVA\|SCALA**
+
+Language tag to instruct Flink runtime how to execute the function. Currently only JAVA and SCALA are supported, the default language for a function is JAVA.
diff --git a/docs/dev/table/sql/create.zh.md b/docs/dev/table/sql/create.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..a69691202243d46f1e3a4625095db138d8e34788
--- /dev/null
+++ b/docs/dev/table/sql/create.zh.md
@@ -0,0 +1,258 @@
+---
+title: "CREATE 语句"
+nav-parent_id: sql
+nav-pos: 2
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+CREATE 语句用于向当前或指定的 [Catalog]({{ site.baseurl }}/zh/dev/table/catalogs.html) 中注册表、视图或函数。注册后的表、视图和函数可以在 SQL 查询中使用。
+
+目前 Flink SQL 支持下列 CREATE 语句：
+
+- CREATE TABLE
+- CREATE DATABASE
+- CREATE FUNCTION
+
+## 执行 CREATE 语句
+
+可以使用 `TableEnvironment` 中的 `sqlUpdate()` 方法执行 CREATE 语句，也可以在 [SQL CLI]({{ site.baseurl }}/zh/dev/table/sqlClient.html) 中执行 CREATE 语句。 若 CREATE 操作执行成功，`sqlUpdate()` 方法不返回任何内容，否则会抛出异常。
+
+以下的例子展示了如何在 `TableEnvironment` 和 SQL CLI 中执行一个 CREATE 语句。
+
+
+
+{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance()...
+TableEnvironment tableEnv = TableEnvironment.create(settings);
+
+// 对已经已经注册的表进行 SQL 查询
+// 注册名为 “Orders” 的表
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+// 在表上执行 SQL 查询，并把得到的结果作为一个新的表
+Table result = tableEnv.sqlQuery(
+ "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+
+// SQL 对已注册的表进行 update 操作
+// 注册 TableSink
+tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)");
+// 在表上执行 SQL 更新查询并向 TableSink 发出结果
+tableEnv.sqlUpdate(
+ "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+val settings = EnvironmentSettings.newInstance()...
+val tableEnv = TableEnvironment.create(settings)
+
+// 对已经已经注册的表进行 SQL 查询
+// 注册名为 “Orders” 的表
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+// 在表上执行 SQL 查询，并把得到的结果作为一个新的表
+val result = tableEnv.sqlQuery(
+ "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+
+// SQL 对已注册的表进行 update 操作
+// 注册 TableSink
+tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH ('connector.path'='/path/to/file' ...)");
+// 在表上执行 SQL 更新查询并向 TableSink 发出结果
+tableEnv.sqlUpdate(
+ "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight python %}
+settings = EnvironmentSettings.newInstance()...
+table_env = TableEnvironment.create(settings)
+
+# 对已经已经注册的表进行 SQL 查询
+# 注册名为 “Orders” 的表
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+# 在表上执行 SQL 查询，并把得到的结果作为一个新的表
+result = tableEnv.sqlQuery(
+ "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+
+# SQL 对已注册的表进行 update 操作
+# 注册 TableSink
+table_env.sql_update("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)")
+# 在表上执行 SQL 更新查询并向 TableSink 发出结果
+table_env \
+ .sql_update("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> CREATE TABLE RubberOrders (product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%';
+[INFO] Submitting SQL update statement to the cluster...
+{% endhighlight %}
+
+
+
+{% top %}
+
+## CREATE TABLE
+
+{% highlight sql %}
+CREATE TABLE [catalog_name.][db_name.]table_name
+ (
+ { | }[ , ...n]
+ [ ]
+ )
+ [COMMENT table_comment]
+ [PARTITIONED BY (partition_column_name1, partition_column_name2, ...)]
+ WITH (key1=val1, key2=val2, ...)
+
+:
+ column_name column_type [COMMENT column_comment]
+
+:
+ column_name AS computed_column_expression [COMMENT column_comment]
+
+:
+ WATERMARK FOR rowtime_column_name AS watermark_strategy_expression
+
+{% endhighlight %}
+
+根据指定的表名创建一个表，如果同名表已经在 catalog 中存在了，则无法注册。
+
+**COMPUTED COLUMN**
+
+计算列是一个使用 “`column_name AS computed_column_expression`” 语法生成的虚拟列。它由使用同一表中其他列的非查询表达式生成，并且不会在表中进行物理存储。例如，一个计算列可以使用 `cost AS price * quantity` 进行定义，这个表达式可以包含物理列、常量、函数或变量的任意组合，但这个表达式不能存在任何子查询。
+
+在 Flink 中计算列一般用于为 CREATE TABLE 语句定义 [时间属性]({{ site.baseurl}}/zh/dev/table/streaming/time_attributes.html)。
+[处理时间属性]({{ site.baseurl}}/zh/dev/table/streaming/time_attributes.html#processing-time) 可以简单地通过使用了系统函数 `PROCTIME()` 的 `proc AS PROCTIME()` 语句进行定义。
+另一方面，由于事件时间列可能需要从现有的字段中获得，因此计算列可用于获得事件时间列。例如，原始字段的类型不是 `TIMESTAMP(3)` 或嵌套在 JSON 字符串中。
+
+注意：
+
+- 定义在一个数据源表（ source table ）上的计算列会在从数据源读取数据后被计算，它们可以在 SELECT 查询语句中使用。
+- 计算列不可以作为 INSERT 语句的目标，在 INSERT 语句中，SELECT 语句的 schema 需要与目标表不带有计算列的 schema 一致。
+
+**WATERMARK**
+
+`WATERMARK` 定义了表的事件时间属性，其形式为 `WATERMARK FOR rowtime_column_name AS watermark_strategy_expression` 。
+
+`rowtime_column_name` 把一个现有的列定义为一个为表标记事件时间的属性。该列的类型必须为 `TIMESTAMP(3)`，且是 schema 中的顶层列，它也可以是一个计算列。
+
+`watermark_strategy_expression` 定义了 watermark 的生成策略。它允许使用包括计算列在内的任意非查询表达式来计算 watermark ；表达式的返回类型必须是 `TIMESTAMP(3)`，表示了从 Epoch 以来的经过的时间。
+返回的 watermark 只有当其不为空且其值大于之前发出的本地 watermark 时才会被发出（以保证 watermark 递增）。每条记录的 watermark 生成表达式计算都会由框架完成。
+框架会定期发出所生成的最大的 watermark ，如果当前 watermark 仍然与前一个 watermark 相同、为空、或返回的 watermark 的值小于最后一个发出的 watermark ，则新的 watermark 不会被发出。
+Watermark 根据 [`pipeline.auto-watermark-interval`]({{ site.baseurl }}/zh/ops/config.html#pipeline-auto-watermark-interval) 中所配置的间隔发出。
+若 watermark 的间隔是 `0ms` ，那么每条记录都会产生一个 watermark，且 watermark 会在不为空并大于上一个发出的 watermark 时发出。
+
+使用事件时间语义时，表必须包含事件时间属性和 watermark 策略。
+
+Flink 提供了几种常用的 watermark 策略。
+
+- 严格递增时间戳： `WATERMARK FOR rowtime_column AS rowtime_column`。
+
+ 发出到目前为止已观察到的最大时间戳的 watermark ，时间戳小于最大时间戳的行被认为没有迟到。
+
+- 递增时间戳： `WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL '0.001' SECOND`。
+
+ 发出到目前为止已观察到的最大时间戳减 1 的 watermark ，时间戳等于或小于最大时间戳的行被认为没有迟到。
+
+- 有界乱序时间戳： `WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL 'string' timeUnit`。
+
+ 发出到目前为止已观察到的最大时间戳减去指定延迟的 watermark ，例如， `WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL '5' SECOND` 是一个 5 秒延迟的 watermark 策略。
+
+{% highlight sql %}
+CREATE TABLE Orders (
+ user BIGINT,
+ product STRING,
+ order_time TIMESTAMP(3),
+ WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND
+) WITH ( . . . );
+{% endhighlight %}
+
+**PARTITIONED BY**
+
+根据指定的列对已经创建的表进行分区。若表使用 filesystem sink ，则将会为每个分区创建一个目录。
+
+**WITH OPTIONS**
+
+表属性用于创建 table source/sink ，一般用于寻找和创建底层的连接器。
+
+表达式 `key1=val1` 的键和值必须为字符串文本常量。请参考 [连接外部系统]({{ site.baseurl }}/zh/dev/table/connect.html) 了解不同连接器所支持的属性。
+
+**注意：** 表名可以为以下三种格式 1. `catalog_name.db_name.table_name` 2. `db_name.table_name` 3. `table_name`。使用`catalog_name.db_name.table_name` 的表将会与名为 "catalog_name" 的 catalog 和名为 "db_name" 的数据库一起注册到 metastore 中。使用 `db_name.table_name` 的表将会被注册到当前执行的 table environment 中的 catalog 且数据库会被命名为 "db_name"；对于 `table_name`, 数据表将会被注册到当前正在运行的catalog和数据库中。
+
+**注意：** 使用 `CREATE TABLE` 语句注册的表均可用作 table source 和 table sink。 在被 DML 语句引用前，我们无法决定其实际用于 source 抑或是 sink。
+
+{% top %}
+
+## CREATE DATABASE
+
+{% highlight sql %}
+CREATE DATABASE [IF NOT EXISTS] [catalog_name.]db_name
+ [COMMENT database_comment]
+ WITH (key1=val1, key2=val2, ...)
+{% endhighlight %}
+
+根据给定的表属性创建数据库。若数据库中已存在同名表会抛出异常。
+
+**IF NOT EXISTS**
+
+若数据库已经存在，则不会进行任何操作。
+
+**WITH OPTIONS**
+
+数据库属性一般用于存储关于这个数据库额外的信息。
+表达式 `key1=val1` 中的键和值都需要是字符串文本常量。
+
+{% top %}
+
+## CREATE FUNCTION
+{% highlight sql%}
+CREATE [TEMPORARY|TEMPORARY SYSTEM] FUNCTION
+ [IF NOT EXISTS] [[catalog_name.]db_name.]function_name
+ AS identifier [LANGUAGE JAVA|SCALA]
+{% endhighlight %}
+
+创建一个有 catalog 和数据库命名空间的 catalog function ，其需要指定 JAVA / SCALA 或其他 language tag 完整的 classpath。 若 catalog 中，已经有同名的函数注册了，则无法注册。
+
+**TEMPORARY**
+
+创建一个有 catalog 和数据库命名空间的临时 catalog function ，并覆盖原有的 catalog function 。
+
+**TEMPORARY SYSTEM**
+
+创建一个没有数据库命名空间的临时系统 catalog function ，并覆盖系统内置的函数。
+
+**IF NOT EXISTS**
+
+若该函数已经存在，则不会进行任何操作。
+
+**LANGUAGE JAVA\|SCALA**
+
+Language tag 用于指定 Flink runtime 如何执行这个函数。目前，只支持 JAVA 和 SCALA，且函数的默认语言为 JAVA。
+
diff --git a/docs/dev/table/sql/drop.md b/docs/dev/table/sql/drop.md
new file mode 100644
index 0000000000000000000000000000000000000000..6f340e8bd446af833f41e61b8159621f1bb22492
--- /dev/null
+++ b/docs/dev/table/sql/drop.md
@@ -0,0 +1,164 @@
+---
+title: "DROP Statements"
+nav-parent_id: sql
+nav-pos: 3
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+DROP statements are used to remove a registered table/view/function from current or specified [Catalog]({{ site.baseurl }}/dev/table/catalogs.html).
+
+Flink SQL supports the following DROP statements for now:
+
+- DROP TABLE
+- DROP DATABASE
+- DROP FUNCTION
+
+## Run a DROP statement
+
+DROP statements can be executed with the `sqlUpdate()` method of the `TableEnvironment`, or executed in [SQL CLI]({{ site.baseurl }}/dev/table/sqlClient.html). The `sqlUpdate()` method returns nothing for a successful DROP operation, otherwise will throw an exception.
+
+The following examples show how to run a DROP statement in `TableEnvironment` and in SQL CLI.
+
+
+
+{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance()...
+TableEnvironment tableEnv = TableEnvironment.create(settings);
+
+// register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+
+// a string array: ["Orders"]
+String[] tables = tableEnv.listTable();
+
+// drop "Orders" table from catalog
+tableEnv.sqlUpdate("DROP TABLE Orders");
+
+// an empty string array
+String[] tables = tableEnv.listTable();
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+val settings = EnvironmentSettings.newInstance()...
+val tableEnv = TableEnvironment.create(settings)
+
+// register a table named "Orders"
+tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)");
+
+// a string array: ["Orders"]
+val tables = tableEnv.listTable()
+
+// drop "Orders" table from catalog
+tableEnv.sqlUpdate("DROP TABLE Orders")
+
+// an empty string array
+val tables = tableEnv.listTable()
+{% endhighlight %}
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> SHOW TABLES;
+Orders
+
+Flink SQL> DROP TABLE Orders;
+[INFO] Table has been removed.
+
+Flink SQL> SHOW TABLES;
+[INFO] Result was empty.
+{% endhighlight %}
+
+
+
+## DROP TABLE
+
+{% highlight sql %}
+DROP TABLE [IF EXISTS] [catalog_name.][db_name.]table_name
+{% endhighlight %}
+
+Drop a table with the given table name. If the table to drop does not exist, an exception is thrown.
+
+**IF EXISTS**
+
+If the table does not exist, nothing happens.
+
+## DROP DATABASE
+
+{% highlight sql %}
+DROP DATABASE [IF EXISTS] [catalog_name.]db_name [ (RESTRICT | CASCADE) ]
+{% endhighlight %}
+
+Drop a database with the given database name. If the database to drop does not exist, an exception is thrown.
+
+**IF EXISTS**
+
+If the database does not exist, nothing happens.
+
+**RESTRICT**
+
+Dropping a non-empty database triggers an exception. Enabled by default.
+
+**CASCADE**
+
+Dropping a non-empty database also drops all associated tables and functions.
+
+## DROP FUNCTION
+
+{% highlight sql%}
+DROP [TEMPORARY|TEMPORARY SYSTEM] FUNCTION [IF EXISTS] [catalog_name.][db_name.]function_name;
+{% endhighlight %}
+
+Drop a catalog function that has catalog and database namespaces. If the function to drop does not exist, an exception is thrown.
+
+**TEMPORARY**
+
+Drop temporary catalog function that has catalog and database namespaces.
+
+**TEMPORARY SYSTEM**
+
+Drop temporary system function that has no namespace.
+
+**IF EXISTS**
+
+If the function doesn't exists, nothing happens.
diff --git a/docs/dev/table/sql/drop.zh.md b/docs/dev/table/sql/drop.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..51ac456c7ad681c81d528f16ec8512bffc60e9f7
--- /dev/null
+++ b/docs/dev/table/sql/drop.zh.md
@@ -0,0 +1,164 @@
+---
+title: "DROP 语句"
+nav-parent_id: sql
+nav-pos: 3
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+DROP 语句用于从当前或指定的 [Catalog]({{ site.baseurl }}/zh/dev/table/catalogs.html) 中删除一个已经注册的表、视图或函数。
+
+Flink SQL 目前支持以下 DROP 语句：
+
+- DROP TABLE
+- DROP DATABASE
+- DROP FUNCTION
+
+## 执行 DROP 语句
+
+可以使用 `TableEnvironment` 中的 `sqlUpdate()` 方法执行 DROP 语句，也可以在 [SQL CLI]({{ site.baseurl }}/zh/dev/table/sqlClient.html) 中执行 DROP 语句。 若 DROP 操作执行成功，`sqlUpdate()` 方法不返回任何内容，否则会抛出异常。
+
+以下的例子展示了如何在 `TableEnvironment` 和 SQL CLI 中执行一个 DROP 语句。
+
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> SHOW TABLES;
+Orders
+
+Flink SQL> DROP TABLE Orders;
+[INFO] Table has been removed.
+
+Flink SQL> SHOW TABLES;
+[INFO] Result was empty.
+{% endhighlight %}
+
+
+
+## DROP TABLE
+
+{% highlight sql %}
+DROP TABLE [IF EXISTS] [catalog_name.][db_name.]table_name
+{% endhighlight %}
+
+根据给定的表名删除某个表。若需要删除的表不存在，则抛出异常。
+
+**IF EXISTS**
+
+表不存在时不会进行任何操作。
+
+## DROP DATABASE
+
+{% highlight sql %}
+DROP DATABASE [IF EXISTS] [catalog_name.]db_name [ (RESTRICT | CASCADE) ]
+{% endhighlight %}
+
+根据给定的表名删除数据库。若需要删除的数据库不存在会抛出异常 。
+
+**IF EXISTS**
+
+若数据库不存在，不执行任何操作。
+
+**RESTRICT**
+
+当删除一个非空数据库时，会触发异常。（默认为开）
+
+**CASCADE**
+
+删除一个非空数据库时，把相关联的表与函数一并删除。
+
+## DROP FUNCTION
+
+{% highlight sql%}
+DROP [TEMPORARY|TEMPORARY SYSTEM] FUNCTION [IF EXISTS] [catalog_name.][db_name.]function_name;
+{% endhighlight %}
+
+删除一个有 catalog 和数据库命名空间的 catalog function。若需要删除的函数不存在，则会产生异常。
+
+**TEMPORARY**
+
+删除一个有 catalog 和数据库命名空间的临时 catalog function。
+
+**TEMPORARY SYSTEM**
+
+删除一个没有数据库命名空间的临时系统函数。
+
+**IF EXISTS**
+
+若函数不存在，则不会进行任何操作。
\ No newline at end of file
diff --git a/docs/dev/table/sql/index.md b/docs/dev/table/sql/index.md
new file mode 100644
index 0000000000000000000000000000000000000000..1affb421cee517ceb0a7b56ad7f70b18fdb107e7
--- /dev/null
+++ b/docs/dev/table/sql/index.md
@@ -0,0 +1,64 @@
+---
+title: "SQL"
+nav-id: sql
+nav-parent_id: tableapi
+nav-pos: 40
+nav-show_overview: true
+---
+
+
+This page describes the SQL language supported in Flink, including Data Definition Language (DDL), Data Manipulation Language (DML) and Query Language. Flink’s SQL support is based on [Apache Calcite](https://calcite.apache.org/) which implements the SQL standard.
+
+This page lists all the supported statements supported in Flink SQL for now:
+
+- [SELECT (Queries)](queries.html)
+- [CREATE TABLE, DATABASE, FUNCTION](create.html)
+- [DROP TABLE, DATABASE, FUNCTION](drop.html)
+- [ALTER TABLE, DATABASE, FUNCTION](alter.html)
+- [INSERT](insert.html)
+
+## Data Types
+
+Please see the dedicated page about [data types]({{ site.baseurl }}/dev/table/types.html).
+
+Generic types and (nested) composite types (e.g., POJOs, tuples, rows, Scala case classes) can be fields of a row as well.
+
+Fields of composite types with arbitrary nesting can be accessed with [value access functions]({{ site.baseurl }}/dev/table/functions/systemFunctions.html#value-access-functions).
+
+Generic types are treated as a black box and can be passed on or processed by [user-defined functions]({{ site.baseurl }}/dev/table/functions/udfs.html).
+
+For DDLs, we support full data types defined in page [Data Types]({{ site.baseurl }}/dev/table/types.html).
+
+**Notes:** Some of the data types are not supported in SQL queries yet (i.e. in cast expressions or literals). E.g. `STRING`, `BYTES`, `RAW`, `TIME(p) WITHOUT TIME ZONE`, `TIME(p) WITH LOCAL TIME ZONE`, `TIMESTAMP(p) WITHOUT TIME ZONE`, `TIMESTAMP(p) WITH LOCAL TIME ZONE`, `ARRAY`, `MULTISET`, `ROW`.
+
+{% top %}
+
+## Reserved Keywords
+
+Although not every SQL feature is implemented yet, some string combinations are already reserved as keywords for future use. If you want to use one of the following strings as a field name, make sure to surround them with backticks (e.g. `` `value` ``, `` `count` ``).
+
+{% highlight sql %}
+
+A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, BYTES, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, RAW, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRING, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE
+
+{% endhighlight %}
+
+{% top %}
+
diff --git a/docs/dev/table/sql/index.zh.md b/docs/dev/table/sql/index.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..5cdc8d0ced07dc182530e8605d4dfddabd1c228e
--- /dev/null
+++ b/docs/dev/table/sql/index.zh.md
@@ -0,0 +1,64 @@
+---
+title: "SQL"
+nav-id: sql
+nav-parent_id: tableapi
+nav-pos: 40
+nav-show_overview: true
+---
+
+
+本页面描述了 Flink 所支持的 SQL 语言，包括数据定义语言（Data Definition Language，DDL）、数据操纵语言（Data Manipulation Language，DML）以及查询语言。Flink 对 SQL 的支持基于实现了 SQL 标准的 [Apache Calcite](https://calcite.apache.org/)。
+
+本页面列出了目前 Flink SQL 所支持的所有语句：
+
+- [SELECT (查询)](queries.html)
+- [CREATE TABLE, DATABASE, FUNCTION](create.html)
+- [DROP TABLE, DATABASE, FUNCTION](drop.html)
+- [ALTER TABLE, DATABASE, FUNCTION](alter.html)
+- [INSERT](insert.html)
+
+## 数据类型
+
+请参考专门描述该主题的页面 [数据类型]({{ site.baseurl }}/zh/dev/table/types.html)。
+
+通用类型与（嵌套的）符合类型 （如：POJO、tuples、rows、Scala case 类) 都可以作为行的字段。
+
+符合类型的字段任意的嵌套可被 [值访问函数]({{ site.baseurl }}/zh/dev/table/functions/systemFunctions.html#value-access-functions) 访问。
+
+通用类型将会被视为一个黑箱，且可以被 [用户自定义函数]({{ site.baseurl }}/zh/dev/table/functions/udfs.html) 传递或引用。
+
+对于 DDL 语句而言，我们支持所有在 [数据类型]({{ site.baseurl }}/zh/dev/table/types.html) 页面中定义的数据类型。
+
+**注意：** SQL查询不支持部分数据类型（cast 表达式或字符常量值）。如：`STRING`, `BYTES`, `RAW`, `TIME(p) WITHOUT TIME ZONE`, `TIME(p) WITH LOCAL TIME ZONE`, `TIMESTAMP(p) WITHOUT TIME ZONE`, `TIMESTAMP(p) WITH LOCAL TIME ZONE`, `ARRAY`, `MULTISET`, `ROW`.
+
+{% top %}
+
+## 保留关键字
+
+虽然 SQL 的特性并未完全实现，但是一些字符串的组合却已经被预留为关键字以备未来使用。如果你希望使用以下字符串作为你的字段名，请在使用时使用反引号将该字段名包起来（如 `` `value` ``, `` `count` `` ）。
+
+{% highlight sql %}
+
+A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, BYTES, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, RAW, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRING, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE
+
+{% endhighlight %}
+
+{% top %}
+
diff --git a/docs/dev/table/sql/insert.md b/docs/dev/table/sql/insert.md
new file mode 100644
index 0000000000000000000000000000000000000000..96052adff5945f4fa0e2258a54c9e484794e32cd
--- /dev/null
+++ b/docs/dev/table/sql/insert.md
@@ -0,0 +1,181 @@
+---
+title: "INSERT Statement"
+nav-parent_id: sql
+nav-pos: 5
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+INSERT statements are used to add rows to a table.
+
+## Run an INSERT statement
+
+INSERT statements are specified with the `sqlUpdate()` method of the `TableEnvironment` or executed in [SQL CLI]({{ site.baseurl }}/dev/table/sqlClient.html). The method `sqlUpdate()` for INSERT statements is a lazy execution, they will be executed only when `TableEnvironment.execute(jobName)` is invoked.
+
+The following examples show how to run an INSERT statement in `TableEnvironment` and in SQL CLI.
+
+
+
+{% highlight java %}
+EnvironmentSettings settings = EnvironmentSettings.newInstance()...
+TableEnvironment tEnv = TableEnvironment.create(settings);
+
+// register a source table named "Orders" and a sink table named "RubberOrders"
+tEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product VARCHAR, amount INT) WITH (...)");
+tEnv.sqlUpdate("CREATE TABLE RubberOrders(product VARCHAR, amount INT) WITH (...)");
+
+// run a SQL update query on the registered source table and emit the result to registered sink table
+tEnv.sqlUpdate(
+ "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+val settings = EnvironmentSettings.newInstance()...
+val tEnv = TableEnvironment.create(settings)
+
+// register a source table named "Orders" and a sink table named "RubberOrders"
+tEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)")
+tEnv.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)")
+
+// run a SQL update query on the registered source table and emit the result to registered sink table
+tEnv.sqlUpdate(
+ "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight python %}
+settings = EnvironmentSettings.newInstance()...
+table_env = TableEnvironment.create(settings)
+
+# register a source table named "Orders" and a sink table named "RubberOrders"
+table_env.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)")
+table_env.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)")
+
+# run a SQL update query on the registered source table and emit the result to registered sink table
+table_env \
+ .sqlUpdate("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...);
+
+Flink SQL> SHOW TABLES;
+Orders
+RubberOrders
+
+Flink SQL> INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%';
+[INFO] Submitting SQL update statement to the cluster...
+[INFO] Table update statement has been successfully submitted to the cluster:
+{% endhighlight %}
+
+
+
+{% top %}
+
+## Insert from select queries
+
+Query Results can be inserted into tables by using the insert clause.
+
+### Syntax
+
+{% highlight sql %}
+
+INSERT { INTO | OVERWRITE } [catalog_name.][db_name.]table_name [PARTITION part_spec] select_statement
+
+part_spec:
+ (part_col_name1=val1 [, part_col_name2=val2, ...])
+
+{% endhighlight %}
+
+**OVERWRITE**
+
+`INSERT OVERWRITE` will overwrite any existing data in the table or partition. Otherwise, new data is appended.
+
+**PARTITION**
+
+`PARTITION` clause should contain static partition columns of this inserting.
+
+### Examples
+
+{% highlight sql %}
+-- Creates a partitioned table
+CREATE TABLE country_page_view (user STRING, cnt INT, date STRING, country STRING)
+PARTITIONED BY (date, country)
+WITH (...)
+
+-- Appends rows into the static partition (date='2019-8-30', country='China')
+INSERT INTO country_page_view PARTITION (date='2019-8-30', country='China')
+ SELECT user, cnt FROM page_view_source;
+
+-- Appends rows into partition (date, country), where date is static partition with value '2019-8-30',
+-- country is dynamic partition whose value is dynamic determined by each row.
+INSERT INTO country_page_view PARTITION (date='2019-8-30')
+ SELECT user, cnt, country FROM page_view_source;
+
+-- Overwrites rows into static partition (date='2019-8-30', country='China')
+INSERT OVERWRITE country_page_view PARTITION (date='2019-8-30', country='China')
+ SELECT user, cnt FROM page_view_source;
+
+-- Overwrites rows into partition (date, country), where date is static partition with value '2019-8-30',
+-- country is dynamic partition whose value is dynamic determined by each row.
+INSERT OVERWRITE country_page_view PARTITION (date='2019-8-30')
+ SELECT user, cnt, country FROM page_view_source;
+{% endhighlight %}
+
+
+## Insert values into tables
+
+The INSERT...VALUES statement can be used to insert data into tables directly from SQL.
+
+### Syntax
+
+{% highlight sql %}
+INSERT { INTO | OVERWRITE } [catalog_name.][db_name.]table_name VALUES values_row [, values_row ...]
+
+values_row:
+ : (val1 [, val2, ...])
+{% endhighlight %}
+
+**OVERWRITE**
+
+`INSERT OVERWRITE` will overwrite any existing data in the table. Otherwise, new data is appended.
+
+### Examples
+
+{% highlight sql %}
+
+CREATE TABLE students (name STRING, age INT, gpa DECIMAL(3, 2)) WITH (...);
+
+INSERT INTO students
+ VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
+
+{% endhighlight %}
+
+{% top %}
\ No newline at end of file
diff --git a/docs/dev/table/sql/insert.zh.md b/docs/dev/table/sql/insert.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..14ad9a4536cdb21c6c13e1651d2f8d51a73ab923
--- /dev/null
+++ b/docs/dev/table/sql/insert.zh.md
@@ -0,0 +1,178 @@
+---
+title: "INSERT 语句"
+nav-parent_id: sql
+nav-pos: 5
+---
+
+
+* This will be replaced by the TOC
+{:toc}
+
+INSERT 语句用来向表中添加行。
+
+## 执行 INSERT 语句
+
+可以使用 `TableEnvironment` 中的 `sqlUpdate()` 方法执行 INSERT 语句，也可以在 [SQL CLI]({{ site.baseurl }}/zh/dev/table/sqlClient.html) 中执行 INSERT 语句。`sqlUpdate()` 方法执行 INSERT 语句时时懒执行的，只有当`TableEnvironment.execute(jobName)`被调用时才会被执行。
+
+以下的例子展示了如何在 `TableEnvironment` 和 SQL CLI 中执行一个 INSERT 语句。
+
+
+{% highlight python %}
+settings = EnvironmentSettings.newInstance()...
+table_env = TableEnvironment.create(settings)
+
+# 注册一个 "Orders" 源表，和 "RubberOrders" 结果表
+table_env.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...)")
+table_env.sqlUpdate("CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...)")
+
+# 运行一个 INSERT 语句，将源表的数据输出到结果表中
+table_env \
+ .sqlUpdate("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
+{% endhighlight %}
+
+
+
+{% highlight sql %}
+Flink SQL> CREATE TABLE Orders (`user` BIGINT, product STRING, amount INT) WITH (...);
+[INFO] Table has been created.
+
+Flink SQL> CREATE TABLE RubberOrders(product STRING, amount INT) WITH (...);
+
+Flink SQL> SHOW TABLES;
+Orders
+RubberOrders
+
+Flink SQL> INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%';
+[INFO] Submitting SQL update statement to the cluster...
+[INFO] Table update statement has been successfully submitted to the cluster:
+{% endhighlight %}
+
+
+
+{% top %}
+
+## 将 SELECT 查询数据插入表中
+
+通过 INSERT 语句，可以将查询的结果插入到表中，
+
+### 语法
+
+{% highlight sql %}
+
+INSERT { INTO | OVERWRITE } [catalog_name.][db_name.]table_name [PARTITION part_spec] select_statement
+
+part_spec:
+ (part_col_name1=val1 [, part_col_name2=val2, ...])
+
+{% endhighlight %}
+
+**OVERWRITE**
+
+`INSERT OVERWRITE` 将会覆盖表中或分区中的任何已存在的数据。否则，新数据会追加到表中或分区中。
+
+**PARTITION**
+
+`PARTITION` 语句应该包含需要插入的静态分区列与值。
+
+### 示例
+
+{% highlight sql %}
+-- 创建一个分区表
+CREATE TABLE country_page_view (user STRING, cnt INT, date STRING, country STRING)
+PARTITIONED BY (date, country)
+WITH (...)
+
+-- 追加行到该静态分区中 (date='2019-8-30', country='China')
+INSERT INTO country_page_view PARTITION (date='2019-8-30', country='China')
+ SELECT user, cnt FROM page_view_source;
+
+-- 追加行到分区 (date, country) 中，其中 date 是静态分区 '2019-8-30'；country 是动态分区，其值由每一行动态决定
+INSERT INTO country_page_view PARTITION (date='2019-8-30')
+ SELECT user, cnt, country FROM page_view_source;
+
+-- 覆盖行到静态分区 (date='2019-8-30', country='China')
+INSERT OVERWRITE country_page_view PARTITION (date='2019-8-30', country='China')
+ SELECT user, cnt FROM page_view_source;
+
+-- 覆盖行到分区 (date, country) 中，其中 date 是静态分区 '2019-8-30'；country 是动态分区，其值由每一行动态决定
+INSERT OVERWRITE country_page_view PARTITION (date='2019-8-30')
+ SELECT user, cnt, country FROM page_view_source;
+{% endhighlight %}
+
+## 将值插入表中
+
+通过 INSERT 语句，也可以直接将值插入到表中，
+
+### 语法
+
+{% highlight sql %}
+INSERT { INTO | OVERWRITE } [catalog_name.][db_name.]table_name VALUES values_row [, values_row ...]
+
+values_row:
+ : (val1 [, val2, ...])
+{% endhighlight %}
+
+**OVERWRITE**
+
+`INSERT OVERWRITE` 将会覆盖表中的任何已存在的数据。否则，新数据会追加到表中。
+
+### 示例
+
+{% highlight sql %}
+
+CREATE TABLE students (name STRING, age INT, gpa DECIMAL(3, 2)) WITH (...);
+
+INSERT INTO students
+ VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
+
+{% endhighlight %}
+
+{% top %}
\ No newline at end of file
diff --git a/docs/dev/table/sql.zh.md b/docs/dev/table/sql/queries.md
similarity index 68%
rename from docs/dev/table/sql.zh.md
rename to docs/dev/table/sql/queries.md
index 84b744e65dd883c0549471fa550d032004aa11eb..de8d90622d0690dc98634a9e5b06bf29d203a986 100644
--- a/docs/dev/table/sql.zh.md
+++ b/docs/dev/table/sql/queries.md
@@ -1,7 +1,7 @@
---
-title: "SQL"
-nav-parent_id: tableapi
-nav-pos: 30
+title: "Queries"
+nav-parent_id: sql
+nav-pos: 1
---
-This is a complete list of Data Definition Language (DDL) and Data Manipulation Language (DML) constructs supported in Flink.
* This will be replaced by the TOC
{:toc}
-## Query
-SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can be seamlessly mixed and are holistically optimized and translated into a single program.
+SELECT queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SELECT query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries]({{ site.baseurl }}/dev/table/common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream]({{ site.baseurl }}/dev/table/common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink]({{ site.baseurl }}/dev/table/common.html#emit-a-table). SQL and Table API queries can be seamlessly mixed and are holistically optimized and translated into a single program.
-In order to access a table in a SQL query, it must be [registered in the TableEnvironment](common.html#register-tables-in-the-catalog). A table can be registered from a [TableSource](common.html#register-a-tablesource), [Table](common.html#register-a-table), [CREATE TABLE statement](#create-table), [DataStream, or DataSet](common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register external catalogs in a TableEnvironment](common.html#register-an-external-catalog) to specify the location of the data sources.
+In order to access a table in a SQL query, it must be [registered in the TableEnvironment]({{ site.baseurl }}/dev/table/common.html#register-tables-in-the-catalog). A table can be registered from a [TableSource]({{ site.baseurl }}/dev/table/common.html#register-a-tablesource), [Table]({{ site.baseurl }}/dev/table/common.html#register-a-table), [CREATE TABLE statement](#create-table), [DataStream, or DataSet]({{ site.baseurl }}/dev/table/common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register catalogs in a TableEnvironment]({{ site.baseurl }}/dev/table/catalogs.html) to specify the location of the data sources.
-For convenience `Table.toString()` automatically registers the table under a unique name in its `TableEnvironment` and returns the name. Hence, `Table` objects can be directly inlined into SQL queries (by string concatenation) as shown in the examples below.
+For convenience, `Table.toString()` automatically registers the table under a unique name in its `TableEnvironment` and returns the name. So, `Table` objects can be directly inlined into SQL queries as shown in the examples below.
-**Note:** Flink's SQL support is not yet feature complete. Queries that include unsupported SQL features cause a `TableException`. The supported features of SQL on batch and streaming tables are listed in the following sections.
+**Note:** Queries that include unsupported SQL features cause a `TableException`. The supported features of SQL on batch and streaming tables are listed in the following sections.
-### Specifying a Query
+## Specifying a Query
The following examples show how to specify a SQL queries on registered and inlined tables.
@@ -54,18 +52,23 @@ Table result = tableEnv.sqlQuery(
"SELECT SUM(amount) FROM " + table + " WHERE product LIKE '%Rubber%'");
// SQL query with a registered table
-// register the DataStream as table "Orders"
-tableEnv.registerDataStream("Orders", ds, "user, product, amount");
+// register the DataStream as view "Orders"
+tableEnv.createTemporaryView("Orders", ds, "user, product, amount");
// run a SQL query on the Table and retrieve the result as a new Table
Table result2 = tableEnv.sqlQuery(
"SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
// SQL update with a registered table
// create and register a TableSink
-TableSink csvSink = new CsvTableSink("/path/to/file", ...);
-String[] fieldNames = {"product", "amount"};
-TypeInformation[] fieldTypes = {Types.STRING, Types.INT};
-tableEnv.registerTableSink("RubberOrders", fieldNames, fieldTypes, csvSink);
+final Schema schema = new Schema()
+ .field("product", DataTypes.STRING())
+ .field("amount", DataTypes.INT());
+
+tableEnv.connect(new FileSystem("/path/to/file"))
+ .withFormat(...)
+ .withSchema(schema)
+ .createTemporaryTable("RubberOrders");
+
// run a SQL update query on the Table and emit the result to the TableSink
tableEnv.sqlUpdate(
"INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
@@ -87,17 +90,22 @@ val result = tableEnv.sqlQuery(
// SQL query with a registered table
// register the DataStream under the name "Orders"
-tableEnv.registerDataStream("Orders", ds, 'user, 'product, 'amount)
+tableEnv.createTemporaryView("Orders", ds, 'user, 'product, 'amount)
// run a SQL query on the Table and retrieve the result as a new Table
val result2 = tableEnv.sqlQuery(
"SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
// SQL update with a registered table
// create and register a TableSink
-val csvSink: CsvTableSink = new CsvTableSink("/path/to/file", ...)
-val fieldNames: Array[String] = Array("product", "amount")
-val fieldTypes: Array[TypeInformation[_]] = Array(Types.STRING, Types.INT)
-tableEnv.registerTableSink("RubberOrders", fieldNames, fieldTypes, csvSink)
+val schema = new Schema()
+ .field("product", DataTypes.STRING())
+ .field("amount", DataTypes.INT())
+
+tableEnv.connect(new FileSystem("/path/to/file"))
+ .withFormat(...)
+ .withSchema(schema)
+ .createTemporaryTable("RubberOrders")
+
// run a SQL update query on the Table and emit the result to the TableSink
tableEnv.sqlUpdate(
"INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
@@ -117,11 +125,15 @@ result = table_env \
# SQL update with a registered table
# create and register a TableSink
-table_env.register_table("Orders", table)
-field_names = ["product", "amount"]
-field_types = [DataTypes.STRING(), DataTypes.BIGINT()]
-csv_sink = CsvTableSink(field_names, field_types, "/path/to/file", ...)
-table_env.register_table_sink("RubberOrders", csv_sink)
+t_env.connect(FileSystem().path("/path/to/file")))
+ .with_format(Csv()
+ .field_delimiter(',')
+ .deriveSchema())
+ .with_schema(Schema()
+ .field("product", DataTypes.STRING())
+ .field("amount", DataTypes.BIGINT()))
+ .create_temporary_table("RubberOrders")
+
# run a SQL update query on the Table and emit the result to the TableSink
table_env \
.sql_update("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
@@ -131,18 +143,13 @@ table_env \
{% top %}
-### Supported Syntax
+## Supported Syntax
-Flink parses SQL using [Apache Calcite](https://calcite.apache.org/docs/reference.html), which supports standard ANSI SQL. DDL statements are not supported by Flink.
+Flink parses SQL using [Apache Calcite](https://calcite.apache.org/docs/reference.html), which supports standard ANSI SQL.
The following BNF-grammar describes the superset of supported SQL features in batch and streaming queries. The [Operations](#operations) section shows examples for the supported features and indicates which features are only supported for batch or streaming queries.
{% highlight sql %}
-
-insert:
- INSERT INTO tableReference
- query
-
query:
values
| {
@@ -276,9 +283,9 @@ String literals must be enclosed in single quotes (e.g., `SELECT 'Hello World'`)
{% top %}
-### Operations
+## Operations
-#### Show and Use
+### Show and Use
@@ -329,7 +336,7 @@ USE mydatabase;
-#### Scan, Projection, and Filter
+### Scan, Projection, and Filter
@@ -372,7 +379,7 @@ SELECT * FROM Orders WHERE a % 2 = 0
BatchStreaming
-
UDFs must be registered in the TableEnvironment. See the UDF documentation for details on how to specify and register scalar UDFs.
+
UDFs must be registered in the TableEnvironment. See the UDF documentation for details on how to specify and register scalar UDFs.
{% highlight sql %}
SELECT PRETTY_PRINT(user) FROM Orders
{% endhighlight %}
@@ -384,7 +391,7 @@ SELECT PRETTY_PRINT(user) FROM Orders
{% top %}
-#### Aggregations
+### Aggregations
@@ -402,7 +409,7 @@ SELECT PRETTY_PRINT(user) FROM Orders
Result Updating
{% highlight sql %}
SELECT a, SUM(b) as d
@@ -431,7 +438,7 @@ GROUP BY TUMBLE(rowtime, INTERVAL '1' DAY), user
Streaming
-
Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute
+
Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute
{% highlight sql %}
SELECT COUNT(amount) OVER (
PARTITION BY user
@@ -458,7 +465,7 @@ WINDOW w AS (
{% highlight sql %}
SELECT DISTINCT users FROM Orders
{% endhighlight %}
-
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
@@ -494,7 +501,7 @@ HAVING SUM(amount) > 50
BatchStreaming
-
UDAGGs must be registered in the TableEnvironment. See the UDF documentation for details on how to specify and register UDAGGs.
+
UDAGGs must be registered in the TableEnvironment. See the UDF documentation for details on how to specify and register UDAGGs.
{% highlight sql %}
SELECT MyAggregate(amount)
FROM Orders
@@ -508,7 +515,7 @@ GROUP BY users
{% top %}
-#### Joins
+### Joins
@@ -531,7 +538,7 @@ GROUP BY users
SELECT *
FROM Orders INNER JOIN Product ON Orders.productId = Product.id
{% endhighlight %}
-
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
@@ -553,7 +560,7 @@ FROM Orders RIGHT JOIN Product ON Orders.productId = Product.id
SELECT *
FROM Orders FULL OUTER JOIN Product ON Orders.productId = Product.id
{% endhighlight %}
-
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
@@ -564,7 +571,7 @@ FROM Orders FULL OUTER JOIN Product ON Orders.productId = Product.id
Note: Time-windowed joins are a subset of regular joins that can be processed in a streaming fashion.
-
A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >), a BETWEEN predicate, or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.
+
A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >), a BETWEEN predicate, or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.
For example, the following predicates are valid window join conditions:
@@ -603,7 +610,7 @@ FROM Orders CROSS JOIN UNNEST(tags) AS t (tag)
Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function.
-
User-defined table functions (UDTFs) must be registered before. See the UDF documentation for details on how to specify and register UDTFs.
+
User-defined table functions (UDTFs) must be registered before. See the UDF documentation for details on how to specify and register UDTFs.
Inner Join
A row of the left (outer) table is dropped, if its table function call returns an empty result.
@@ -628,13 +635,13 @@ FROM Orders LEFT JOIN LATERAL TABLE(unnest_udtf(tags)) t AS tag ON TRUE
Streaming
A Temporal table function provides access to the state of a temporal table at a specific point in time.
The syntax to join a table with a temporal table function is the same as in Join with Table Function.
Note: Currently only inner joins with temporal tables are supported.
Temporal Tables are tables that track changes over time.
- A Temporal Table provides access to the versions of a temporal table at a specific point in time.
+
Temporal Tables are tables that track changes over time.
+ A Temporal Table provides access to the versions of a temporal table at a specific point in time.
Only inner and left joins with processing-time temporal tables are supported.
-
The following example assumes that LatestRates is a Temporal Table which is materialized with the latest rate.
+
The following example assumes that LatestRates is a Temporal Table which is materialized with the latest rate.
{% highlight sql %}
SELECT
o.amout, o.currency, r.rate, o.amount * r.rate
@@ -666,7 +673,7 @@ FROM
JOIN LatestRates FOR SYSTEM_TIME AS OF o.proctime AS r
ON r.currency = o.currency
{% endhighlight %}
-
For more information please check the more detailed Temporal Tables concept description.
+
For more information please check the more detailed Temporal Tables concept description.
Only supported in Blink planner.
@@ -677,7 +684,7 @@ FROM
{% top %}
-#### Set Operations
+### Set Operations
@@ -760,7 +767,7 @@ WHERE product IN (
SELECT product FROM NewProducts
)
{% endhighlight %}
-
Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
@@ -778,7 +785,7 @@ WHERE product EXISTS (
SELECT product FROM NewProducts
)
{% endhighlight %}
-
Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
@@ -787,7 +794,7 @@ WHERE product EXISTS (
{% top %}
-#### OrderBy & Limit
+### OrderBy & Limit
@@ -804,7 +811,7 @@ WHERE product EXISTS (
BatchStreaming
-Note: The result of streaming queries must be primarily sorted on an ascending time attribute. Additional sorting attributes are supported.
+Note: The result of streaming queries must be primarily sorted on an ascending time attribute. Additional sorting attributes are supported.
{% highlight sql %}
SELECT *
@@ -835,7 +842,7 @@ LIMIT 3
{% top %}
-#### Top-N
+### Top-N
Attention Top-N is only supported in Blink planner.
@@ -880,7 +887,7 @@ StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
// ingest a DataStream from an external source
DataStream> ds = env.addSource(...);
// register the DataStream as table "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, "product_id, category, product_name, sales");
+tableEnv.createTemporaryView("ShopSales", ds, "product_id, category, product_name, sales");
// select top-5 products per category which have the maximum sales.
Table result1 = tableEnv.sqlQuery(
@@ -901,7 +908,7 @@ val tableEnv = TableEnvironment.getTableEnvironment(env)
// read a DataStream from an external source
val ds: DataStream[(String, String, String, Long)] = env.addSource(...)
// register the DataStream under the name "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
+tableEnv.createTemporaryView("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
// select top-5 products per category which have the maximum sales.
@@ -918,7 +925,7 @@ val result1 = tableEnv.sqlQuery(
-##### No Ranking Output Optimization
+#### No Ranking Output Optimization
As described above, the `rownum` field will be written into the result table as one field of the unique key, which may lead to a lot of records being written to the result table. For example, when the record (say `product-1001`) of ranking 9 is updated and its rank is upgraded to 1, all the records from ranking 1 ~ 9 will be output to the result table as update messages. If the result table receives too many data, it will become the bottleneck of the SQL job.
@@ -935,7 +942,7 @@ StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
// ingest a DataStream from an external source
DataStream> ds = env.addSource(...);
// register the DataStream as table "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, "product_id, category, product_name, sales");
+tableEnv.createTemporaryView("ShopSales", ds, "product_id, category, product_name, sales");
// select top-5 products per category which have the maximum sales.
Table result1 = tableEnv.sqlQuery(
@@ -956,7 +963,7 @@ val tableEnv = TableEnvironment.getTableEnvironment(env)
// read a DataStream from an external source
val ds: DataStream[(String, String, String, Long)] = env.addSource(...)
// register the DataStream under the name "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
+tableEnv.createTemporaryView("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
// select top-5 products per category which have the maximum sales.
@@ -977,7 +984,7 @@ val result1 = tableEnv.sqlQuery(
{% top %}
-#### Deduplication
+### Deduplication
Attention Deduplication is only supported in Blink planner.
@@ -1001,7 +1008,7 @@ WHERE rownum = 1
- `ROW_NUMBER()`: Assigns an unique, sequential number to each row, starting with one.
- `PARTITION BY col1[, col2...]`: Specifies the partition columns, i.e. the deduplicate key.
-- `ORDER BY time_attr [asc|desc]`: Specifies the ordering column, it must be a [time attribute](streaming/time_attributes.html). Currently only support [proctime attribute](streaming/time_attributes.html#processing-time). [Rowtime atttribute](streaming/time_attributes.html#event-time) will be supported in the future. Ordering by ASC means keeping the first row, ordering by DESC means keeping the last row.
+- `ORDER BY time_attr [asc|desc]`: Specifies the ordering column, it must be a [time attribute]({{ site.baseurl }}/dev/table/streaming/time_attributes.html). Currently only support [proctime attribute]({{ site.baseurl }}/dev/table/streaming/time_attributes.html#processing-time). [Rowtime atttribute]({{ site.baseurl }}/dev/table/streaming/time_attributes.html#event-time) will be supported in the future. Ordering by ASC means keeping the first row, ordering by DESC means keeping the last row.
- `WHERE rownum = 1`: The `rownum = 1` is required for Flink to recognize this query is deduplication.
The following examples show how to specify SQL queries with Deduplication on streaming tables.
@@ -1015,7 +1022,7 @@ StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
// ingest a DataStream from an external source
DataStream> ds = env.addSource(...);
// register the DataStream as table "Orders"
-tableEnv.registerDataStream("Orders", ds, "order_id, user, product, number, proctime.proctime");
+tableEnv.createTemporaryView("Orders", ds, "order_id, user, product, number, proctime.proctime");
// remove duplicate rows on order_id and keep the first occurrence row,
// because there shouldn't be two orders with the same order_id.
@@ -1037,7 +1044,7 @@ val tableEnv = TableEnvironment.getTableEnvironment(env)
// read a DataStream from an external source
val ds: DataStream[(String, String, String, Int)] = env.addSource(...)
// register the DataStream under the name "Orders"
-tableEnv.registerDataStream("Orders", ds, 'order_id, 'user, 'product, 'number, 'proctime.proctime)
+tableEnv.createTemporaryView("Orders", ds, 'order_id, 'user, 'product, 'number, 'proctime.proctime)
// remove duplicate rows on order_id and keep the first occurrence row,
// because there shouldn't be two orders with the same order_id.
@@ -1056,40 +1063,7 @@ val result1 = tableEnv.sqlQuery(
{% top %}
-#### Insert
-
-
-
-
-
-
Operation
-
Description
-
-
-
-
-
- Insert Into
- BatchStreaming
-
-
-
Output tables must be registered in the TableEnvironment (see Register a TableSink). Moreover, the schema of the registered table must match the schema of the query.
-
-{% highlight sql %}
-INSERT INTO OutputTable
-SELECT users, tag
-FROM Orders
-{% endhighlight %}
-
-
-
-
-
-
-
-{% top %}
-
-#### Group Windows
+### Group Windows
Group windows are defined in the `GROUP BY` clause of a SQL query. Just like queries with regular `GROUP BY` clauses, queries with a `GROUP BY` clause that includes a group window function compute a single result row per group. The following group windows functions are supported for SQL on batch and streaming tables.
@@ -1117,13 +1091,13 @@ Group windows are defined in the `GROUP BY` clause of a SQL query. Just like que
-##### Time Attributes
+#### Time Attributes
-For SQL queries on streaming tables, the `time_attr` argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows. See the [documentation of time attributes](streaming/time_attributes.html) to learn how to define time attributes.
+For SQL queries on streaming tables, the `time_attr` argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows. See the [documentation of time attributes]({{ site.baseurl }}/dev/table/streaming/time_attributes.html) to learn how to define time attributes.
For SQL on batch tables, the `time_attr` argument of the group window function must be an attribute of type `TIMESTAMP`.
-##### Selecting Group Window Start and End Timestamps
+#### Selecting Group Window Start and End Timestamps
The start and end timestamps of group windows as well as time attributes can be selected with the following auxiliary functions:
@@ -1151,7 +1125,7 @@ The start and end timestamps of group windows as well as time attributes can be
SESSION_END(time_attr, interval)
Returns the timestamp of the exclusive upper bound of the corresponding tumbling, hopping, or session window.
@@ -1168,7 +1142,7 @@ The start and end timestamps of group windows as well as time attributes can be
HOP_PROCTIME(time_attr, interval, interval) SESSION_PROCTIME(time_attr, interval)
@@ -1186,7 +1160,7 @@ StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
// ingest a DataStream from an external source
DataStream> ds = env.addSource(...);
// register the DataStream as table "Orders"
-tableEnv.registerDataStream("Orders", ds, "user, product, amount, proctime.proctime, rowtime.rowtime");
+tableEnv.createTemporaryView("Orders", ds, "user, product, amount, proctime.proctime, rowtime.rowtime");
// compute SUM(amount) per day (in event-time)
Table result1 = tableEnv.sqlQuery(
@@ -1223,7 +1197,7 @@ val tableEnv = StreamTableEnvironment.create(env)
// read a DataStream from an external source
val ds: DataStream[(Long, String, Int)] = env.addSource(...)
// register the DataStream under the name "Orders"
-tableEnv.registerDataStream("Orders", ds, 'user, 'product, 'amount, 'proctime.proctime, 'rowtime.rowtime)
+tableEnv.createTemporaryView("Orders", ds, 'user, 'product, 'amount, 'proctime.proctime, 'rowtime.rowtime)
// compute SUM(amount) per day (in event-time)
val result1 = tableEnv.sqlQuery(
@@ -1262,7 +1236,7 @@ val result4 = tableEnv.sqlQuery(
{% top %}
-#### Pattern Recognition
+### Pattern Recognition
@@ -1280,7 +1254,7 @@ val result4 = tableEnv.sqlQuery(
Searches for a given pattern in a streaming table according to the MATCH_RECOGNIZEISO standard. This makes it possible to express complex event processing (CEP) logic in SQL queries.
{% highlight sql %}
SELECT T.aid, T.bid, T.cid
@@ -1307,144 +1281,3 @@ MATCH_RECOGNIZE (
{% top %}
-
-## DDL
-
-DDLs are specified with the `sqlUpdate()` method of the `TableEnvironment`. The method returns nothing for a success table creation. A `Table` can be register into the [Catalog](catalogs.html) with a `CREATE TABLE` statement, then can be referenced in SQL queries in method `sqlQuery()` of `TableEnvironment`.
-
-**Note:** Flink's DDL support is not yet feature complete. Queries that include unsupported SQL features cause a `TableException`. The supported features of SQL DDL on batch and streaming tables are listed in the following sections.
-
-### Specifying a DDL
-
-The following examples show how to specify a SQL DDL.
-
-
-
-{% highlight java %}
-StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
-
-// SQL query with a registered table
-// register a table named "Orders"
-tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product VARCHAR, amount INT) WITH (...)");
-// run a SQL query on the Table and retrieve the result as a new Table
-Table result = tableEnv.sqlQuery(
- "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-
-// SQL update with a registered table
-// register a TableSink
-tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product VARCHAR, amount INT) WITH (...)");
-// run a SQL update query on the Table and emit the result to the TableSink
-tableEnv.sqlUpdate(
- "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-{% endhighlight %}
-
-
-
-{% highlight scala %}
-val env = StreamExecutionEnvironment.getExecutionEnvironment
-val tableEnv = StreamTableEnvironment.create(env)
-
-// SQL query with a registered table
-// register a table named "Orders"
-tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product VARCHAR, amount INT) WITH (...)");
-// run a SQL query on the Table and retrieve the result as a new Table
-val result = tableEnv.sqlQuery(
- "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-
-// SQL update with a registered table
-// register a TableSink
-tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product VARCHAR, amount INT) WITH ('connector.path'='/path/to/file' ...)");
-// run a SQL update query on the Table and emit the result to the TableSink
-tableEnv.sqlUpdate(
- "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
-{% endhighlight %}
-
-
-
-{% highlight python %}
-env = StreamExecutionEnvironment.get_execution_environment()
-table_env = StreamTableEnvironment.create(env)
-
-# SQL update with a registered table
-# register a TableSink
-table_env.sql_update("CREATE TABLE RubberOrders(product VARCHAR, amount INT) with (...)")
-# run a SQL update query on the Table and emit the result to the TableSink
-table_env \
- .sql_update("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
-{% endhighlight %}
-
-
-
-{% top %}
-
-### Create Table
-
-{% highlight sql %}
-CREATE TABLE [catalog_name.][db_name.]table_name
- [(col_name1 col_type1 [COMMENT col_comment1], ...)]
- [COMMENT table_comment]
- [PARTITIONED BY (col_name1, col_name2, ...)]
- WITH (key1=val1, key2=val2, ...)
-{% endhighlight %}
-
-Create a table with the given table properties. If a table with the same name already exists in the database, an exception is thrown.
-
-**PARTITIONED BY**
-
-Partition the created table by the specified columns. A directory is created for each partition if this table is used as a filesystem sink.
-
-**WITH OPTIONS**
-
-Table properties used to create a table source/sink. The properties are usually used to find and create the underlying connector.
-
-The key and value of expression `key1=val1` should both be string literal. See details in [Connect to External Systems](connect.html) for all the supported table properties of different connectors.
-
-**Notes:** The table name can be of three formats: 1. `catalog_name.db_name.table_name` 2. `db_name.table_name` 3. `table_name`. For `catalog_name.db_name.table_name`, the table would be registered into metastore with catalog named "catalog_name" and database named "db_name"; for `db_name.table_name`, the table would be registered into the current catalog of the execution table environment and database named "db_name"; for `table_name`, the table would be registered into the current catalog and database of the execution table environment.
-
-**Notes:** The table registered with `CREATE TABLE` statement can be used as both table source and table sink, we can not decide if it is used as a source or sink until it is referenced in the DMLs.
-
-{% top %}
-
-### Drop Table
-
-{% highlight sql %}
-DROP TABLE [IF EXISTS] [catalog_name.][db_name.]table_name
-{% endhighlight %}
-
-Drop a table with the given table name. If the table to drop does not exist, an exception is thrown.
-
-**IF EXISTS**
-
-If the table does not exist, nothing happens.
-
-{% top %}
-
-## Data Types
-
-Please see the dedicated page about [data types](types.html).
-
-Generic types and (nested) composite types (e.g., POJOs, tuples, rows, Scala case classes) can be fields of a row as well.
-
-Fields of composite types with arbitrary nesting can be accessed with [value access functions](functions.html#value-access-functions).
-
-Generic types are treated as a black box and can be passed on or processed by [user-defined functions](udfs.html).
-
-For DDLs, we support full data types defined in page [Data Types]({{ site.baseurl }}/dev/table/types.html).
-
-**Notes:** Some of the data types are not supported in the sql query(the cast expression or literals). E.G. `STRING`, `BYTES`, `TIME(p) WITHOUT TIME ZONE`, `TIME(p) WITH LOCAL TIME ZONE`, `TIMESTAMP(p) WITHOUT TIME ZONE`, `TIMESTAMP(p) WITH LOCAL TIME ZONE`, `ARRAY`, `MULTISET`, `ROW`.
-
-{% top %}
-
-## Reserved Keywords
-
-Although not every SQL feature is implemented yet, some string combinations are already reserved as keywords for future use. If you want to use one of the following strings as a field name, make sure to surround them with backticks (e.g. `` `value` ``, `` `count` ``).
-
-{% highlight sql %}
-
-A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, BYTES, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRING, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE
-
-{% endhighlight %}
-
-{% top %}
-
diff --git a/docs/dev/table/sql.md b/docs/dev/table/sql/queries.zh.md
similarity index 33%
rename from docs/dev/table/sql.md
rename to docs/dev/table/sql/queries.zh.md
index c3d0cd39f095037bda4e1a548a873b59e8e8f381..0d68ae4671ac7dc2ec75951752ba9e220e1245ce 100644
--- a/docs/dev/table/sql.md
+++ b/docs/dev/table/sql/queries.zh.md
@@ -1,7 +1,7 @@
---
-title: "SQL"
-nav-parent_id: tableapi
-nav-pos: 30
+title: "查询语句"
+nav-parent_id: sql
+nav-pos: 1
---
-This is a complete list of Data Definition Language (DDL) and Data Manipulation Language (DML) constructs supported in Flink.
* This will be replaced by the TOC
{:toc}
-## Query
-SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can be seamlessly mixed and are holistically optimized and translated into a single program.
+SELECT 查询需要使用 `TableEnvironment` 的 `sqlQuery()` 方法加以指定。这个方法会以 `Table` 的形式返回 SELECT 的查询结果。 `Table` 可以被用于 [随后的SQL 与 Table API 查询]({{ site.baseurl }}/zh/dev/table/common.html#mixing-table-api-and-sql) 、 [转换为 DataSet 或 DataStream ]({{ site.baseurl }}/zh/dev/table/common.html#integration-with-datastream-and-dataset-api)或 [输出到 TableSink ]({{ site.baseurl }}/zh/dev/table/common.html#emit-a-table)。SQL 与 Table API 的查询可以进行无缝融合、整体优化并翻译为单一的程序。
-In order to access a table in a SQL query, it must be [registered in the TableEnvironment](common.html#register-tables-in-the-catalog). A table can be registered from a [TableSource](common.html#register-a-tablesource), [Table](common.html#register-a-table), [CREATE TABLE statement](#create-table), [DataStream, or DataSet](common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register external catalogs in a TableEnvironment](common.html#register-an-external-catalog) to specify the location of the data sources.
+为了可以在 SQL 查询中访问到表，你需要先 [在 TableEnvironment 中注册表 ]({{ site.baseurl }}/zh/dev/table/common.html#register-tables-in-the-catalog)。表可以通过 [TableSource]({{ site.baseurl }}/zh/dev/table/common.html#register-a-tablesource)、 [Table]({{ site.baseurl }}/zh/dev/table/common.html#register-a-table)、[CREATE TABLE 语句](create.html)、 [DataStream 或 DataSet]({{ site.baseurl }}/zh/dev/table/common.html#register-a-datastream-or-dataset-as-table) 注册。 用户也可以通过 [向 TableEnvironment 中注册 catalog ]({{ site.baseurl }}/zh/dev/table/catalogs.html) 的方式指定数据源的位置。
-For convenience `Table.toString()` automatically registers the table under a unique name in its `TableEnvironment` and returns the name. Hence, `Table` objects can be directly inlined into SQL queries (by string concatenation) as shown in the examples below.
+为方便起见 `Table.toString()` 将会在其 `TableEnvironment` 中自动使用一个唯一的名字注册表并返回表名。 因此， `Table` 对象可以如下文所示样例，直接内联到 SQL 查询中。
-**Note:** Flink's SQL support is not yet feature complete. Queries that include unsupported SQL features cause a `TableException`. The supported features of SQL on batch and streaming tables are listed in the following sections.
+**注意：** 查询若包括了不支持的 SQL 特性，将会抛出 `TableException`。批处理和流处理所支持的 SQL 特性将会在下述章节中列出。
-### Specifying a Query
+## 指定查询
-The following examples show how to specify a SQL queries on registered and inlined tables.
+以下示例显示如何在已注册和内联表上指定 SQL 查询。
@@ -45,28 +43,33 @@ The following examples show how to specify a SQL queries on registered and inlin
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
-// ingest a DataStream from an external source
+// 从外部数据源读取 DataStream
DataStream> ds = env.addSource(...);
-// SQL query with an inlined (unregistered) table
+// 使用 SQL 查询内联的（未注册的）表
Table table = tableEnv.fromDataStream(ds, "user, product, amount");
Table result = tableEnv.sqlQuery(
"SELECT SUM(amount) FROM " + table + " WHERE product LIKE '%Rubber%'");
-// SQL query with a registered table
-// register the DataStream as table "Orders"
-tableEnv.registerDataStream("Orders", ds, "user, product, amount");
-// run a SQL query on the Table and retrieve the result as a new Table
+// SQL 查询一个已经注册的表
+// 根据视图 "Orders" 创建一个 DataStream
+tableEnv.createTemporaryView("Orders", ds, "user, product, amount");
+// 在表上执行 SQL 查询并得到以新表返回的结果
Table result2 = tableEnv.sqlQuery(
"SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-// SQL update with a registered table
-// create and register a TableSink
-TableSink csvSink = new CsvTableSink("/path/to/file", ...);
-String[] fieldNames = {"product", "amount"};
-TypeInformation[] fieldTypes = {Types.STRING, Types.INT};
-tableEnv.registerTableSink("RubberOrders", fieldNames, fieldTypes, csvSink);
-// run a SQL update query on the Table and emit the result to the TableSink
+// SQL 更新一个已经注册的表
+// 创建并注册一个 TableSink
+final Schema schema = new Schema()
+ .field("product", DataTypes.STRING())
+ .field("amount", DataTypes.INT());
+
+tableEnv.connect(new FileSystem("/path/to/file"))
+ .withFormat(...)
+ .withSchema(schema)
+ .createTemporaryTable("RubberOrders");
+
+// 在表上执行更新语句并把结果发出到 TableSink
tableEnv.sqlUpdate(
"INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
{% endhighlight %}
@@ -77,28 +80,33 @@ tableEnv.sqlUpdate(
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tableEnv = StreamTableEnvironment.create(env)
-// read a DataStream from an external source
+// 从外部数据源读取 DataStream
val ds: DataStream[(Long, String, Integer)] = env.addSource(...)
-// SQL query with an inlined (unregistered) table
+// 使用 SQL 查询内联的（未注册的）表
val table = ds.toTable(tableEnv, 'user, 'product, 'amount)
val result = tableEnv.sqlQuery(
s"SELECT SUM(amount) FROM $table WHERE product LIKE '%Rubber%'")
-// SQL query with a registered table
-// register the DataStream under the name "Orders"
-tableEnv.registerDataStream("Orders", ds, 'user, 'product, 'amount)
-// run a SQL query on the Table and retrieve the result as a new Table
+// SQL 查询一个已经注册的表
+// 使用名称 "Orders" 注册一个 DataStream
+tableEnv.createTemporaryView("Orders", ds, 'user, 'product, 'amount)
+// 在表上执行 SQL 查询并得到以新表返回的结果
val result2 = tableEnv.sqlQuery(
"SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
-// SQL update with a registered table
-// create and register a TableSink
-val csvSink: CsvTableSink = new CsvTableSink("/path/to/file", ...)
-val fieldNames: Array[String] = Array("product", "amount")
-val fieldTypes: Array[TypeInformation[_]] = Array(Types.STRING, Types.INT)
-tableEnv.registerTableSink("RubberOrders", fieldNames, fieldTypes, csvSink)
-// run a SQL update query on the Table and emit the result to the TableSink
+// 使用 SQL 更新一个已经注册的表
+// 创建并注册一个 TableSink
+val schema = new Schema()
+ .field("product", DataTypes.STRING())
+ .field("amount", DataTypes.INT())
+
+tableEnv.connect(new FileSystem("/path/to/file"))
+ .withFormat(...)
+ .withSchema(schema)
+ .createTemporaryTable("RubberOrders")
+
+// 在表上执行 SQL 更新操作，并把结果发出到 TableSink
tableEnv.sqlUpdate(
"INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
{% endhighlight %}
@@ -109,20 +117,24 @@ tableEnv.sqlUpdate(
env = StreamExecutionEnvironment.get_execution_environment()
table_env = StreamTableEnvironment.create(env)
-# SQL query with an inlined (unregistered) table
-# elements data type: BIGINT, STRING, BIGINT
+# SQL 查询内联的（未注册的）表
+# 元素数据类型： BIGINT, STRING, BIGINT
table = table_env.from_elements(..., ['user', 'product', 'amount'])
result = table_env \
.sql_query("SELECT SUM(amount) FROM %s WHERE product LIKE '%%Rubber%%'" % table)
-# SQL update with a registered table
-# create and register a TableSink
-table_env.register_table("Orders", table)
-field_names = ["product", "amount"]
-field_types = [DataTypes.STRING(), DataTypes.BIGINT()]
-csv_sink = CsvTableSink(field_names, field_types, "/path/to/file", ...)
-table_env.register_table_sink("RubberOrders", csv_sink)
-# run a SQL update query on the Table and emit the result to the TableSink
+# SQL 更新已经注册的表
+# 创建并注册 TableSink
+t_env.connect(FileSystem().path("/path/to/file")))
+ .with_format(Csv()
+ .field_delimiter(',')
+ .deriveSchema())
+ .with_schema(Schema()
+ .field("product", DataTypes.STRING())
+ .field("amount", DataTypes.BIGINT()))
+ .create_temporary_table("RubberOrders")
+
+# 在表上执行 SQL 更新操作，并把结果发出到 TableSink
table_env \
.sql_update("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
{% endhighlight %}
@@ -131,18 +143,13 @@ table_env \
{% top %}
-### Supported Syntax
+## 支持的语法
-Flink parses SQL using [Apache Calcite](https://calcite.apache.org/docs/reference.html), which supports standard ANSI SQL. DDL statements are not supported by Flink.
+Flink 通过支持标准 ANSI SQL的 [Apache Calcite](https://calcite.apache.org/docs/reference.html) 解析 SQL。
-The following BNF-grammar describes the superset of supported SQL features in batch and streaming queries. The [Operations](#operations) section shows examples for the supported features and indicates which features are only supported for batch or streaming queries.
+以下 BNF-语法 描述了批处理和流处理查询中所支持的 SQL 特性的超集。其中 [操作符](#操作符) 章节展示了所支持的特性的样例，并指明了哪些特性仅适用于批处理或流处理。
{% highlight sql %}
-
-insert:
- INSERT INTO tableReference
- query
-
query:
values
| {
@@ -168,7 +175,7 @@ select:
[ GROUP BY { groupItem [, groupItem ]* } ]
[ HAVING booleanExpression ]
[ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ]
-
+
selectWithoutFrom:
SELECT [ ALL | DISTINCT ]
{ * | projectItem [, projectItem ]* }
@@ -263,47 +270,47 @@ patternQuantifier:
{% endhighlight %}
-Flink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:
+Flink SQL 对于标识符（表、属性、函数名）有类似于 Java 的词法约定:
-- The case of identifiers is preserved whether or not they are quoted.
-- After which, identifiers are matched case-sensitively.
-- Unlike Java, back-ticks allow identifiers to contain non-alphanumeric characters (e.g. "SELECT a AS `my field` FROM t").
+- 不管是否引用标识符，都保留标识符的大小写。
+- 且标识符需区分大小写。
+- 与 Java 不一样的地方在于，通过反引号，可以允许标识符带有非字母的字符（如："SELECT a AS `my field` FROM t"）。
-String literals must be enclosed in single quotes (e.g., `SELECT 'Hello World'`). Duplicate a single quote for escaping (e.g., `SELECT 'It''s me.'`). Unicode characters are supported in string literals. If explicit unicode code points are required, use the following syntax:
+字符串文本常量需要被单引号包起来（如 `SELECT 'Hello World'` ）。两个单引号表示转移（如 `SELECT 'It''s me.'`）。字符串文本常量支持 Unicode 字符，如需明确使用 Unicode 编码，请使用以下语法：
-- Use the backslash (`\`) as escaping character (default): `SELECT U&'\263A'`
-- Use a custom escaping character: `SELECT U&'#263A' UESCAPE '#'`
+- 使用反斜杠（`\`）作为转义字符（默认）：`SELECT U&'\263A'`
+- 使用自定义的转义字符： `SELECT U&'#263A' UESCAPE '#'`
{% top %}
-### Operations
+## 操作符
-#### Show and Use
+### Show 与 Use
-
Operation
-
Description
+
操作符
+
描述
Show
- BatchStreaming
+ 批处理流处理
-
Show all catalogs
+
显示所有 catalog
{% highlight sql %}
SHOW CATALOGS;
{% endhighlight %}
-
Show all databases in the current catalog
+
显示当前 catalog 中所有的数据库
{% highlight sql %}
SHOW DATABASES;
{% endhighlight %}
-
Show all tables in the current database in the current catalog
+
显示当前数据库、Catalog中的所有表
{% highlight sql %}
SHOW TABLES;
{% endhighlight %}
@@ -312,14 +319,14 @@ SHOW TABLES;
{% highlight sql %}
SELECT user, SUM(amount)
FROM Orders
@@ -426,12 +433,12 @@ GROUP BY TUMBLE(rowtime, INTERVAL '1' DAY), user
-
+
Over Window aggregation
- Streaming
+ 流处理
-
-
Note: All aggregates must be defined over the same window, i.e., same partitioning, sorting, and range. Currently, only windows with PRECEDING (UNBOUNDED and bounded) to CURRENT ROW range are supported. Ranges with FOLLOWING are not supported yet. ORDER BY must be specified on a single time attribute
+
+
注意： 所有的聚合必须定义到同一个窗口中，即相同的分区、排序和区间。当前仅支持 PRECEDING (无界或有界) 到 CURRENT ROW 范围内的窗口、FOLLOWING 所描述的区间并未支持，ORDER BY 必须指定于单个的时间属性。
{% highlight sql %}
SELECT COUNT(amount) OVER (
PARTITION BY user
@@ -451,20 +458,20 @@ WINDOW w AS (
Distinct
- BatchStreaming
- Result Updating
+ 批处理流处理
+ 可自动更新结果
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct fields. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
{% highlight sql %}
SELECT MyAggregate(amount)
FROM Orders
@@ -508,41 +515,41 @@ GROUP BY users
{% top %}
-#### Joins
+### Joins
-
Operation
-
Description
+
操作符
+
描述
Inner Equi-join
- Batch
- Streaming
+ 批处理
+ 流处理
-
Currently, only equi-joins are supported, i.e., joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or theta joins are not supported.
-
Note: The order of joins is not optimized. Tables are joined in the order in which they are specified in the FROM clause. Make sure to specify tables in an order that does not yield a cross join (Cartesian product) which are not supported and would cause a query to fail.
{% highlight sql %}
SELECT *
FROM Orders INNER JOIN Product ON Orders.productId = Product.id
{% endhighlight %}
-
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
Currently, only equi-joins are supported, i.e., joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or theta joins are not supported.
-
Note: The order of joins is not optimized. Tables are joined in the order in which they are specified in the FROM clause. Make sure to specify tables in an order that does not yield a cross join (Cartesian product) which are not supported and would cause a query to fail.
{% highlight sql %}
SELECT *
FROM Orders LEFT JOIN Product ON Orders.productId = Product.id
@@ -553,19 +560,19 @@ FROM Orders RIGHT JOIN Product ON Orders.productId = Product.id
SELECT *
FROM Orders FULL OUTER JOIN Product ON Orders.productId = Product.id
{% endhighlight %}
-
Note: For streaming queries the required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
Note: Time-windowed joins are a subset of regular joins that can be processed in a streaming fashion.
+
注意： 时间窗口 join 是常规 join 的子集，可以使用流的方式进行处理。
-
A time-windowed join requires at least one equi-join predicate and a join condition that bounds the time on both sides. Such a condition can be defined by two appropriate range predicates (<, <=, >=, >), a BETWEEN predicate, or a single equality predicate that compares time attributes of the same type (i.e., processing time or event time) of both input tables.
-
For example, the following predicates are valid window join conditions:
@@ -580,16 +587,16 @@ WHERE o.id = s.orderId AND
o.ordertime BETWEEN s.shiptime - INTERVAL '4' HOUR AND s.shiptime
{% endhighlight %}
-The example above will join all orders with their corresponding shipments if the order was shipped four hours after the order was received.
+以上示例中，所有在收到后四小时内发货的 order 会与他们相关的 shipment 进行 join。
-
+
Expanding arrays into a relation
- BatchStreaming
+ 批处理流处理
-
-
Unnesting WITH ORDINALITY is not supported yet.
+
+
目前尚未支持非嵌套的 WITH ORDINALITY 。
{% highlight sql %}
SELECT users, tag
FROM Orders CROSS JOIN UNNEST(tags) AS t (tag)
@@ -597,44 +604,44 @@ FROM Orders CROSS JOIN UNNEST(tags) AS t (tag)
-
- Join with Table Function (UDTF)
- BatchStreaming
+
+ Join 表函数 (UDTF)
+ 批处理流处理
-
-
Joins a table with the results of a table function. Each row of the left (outer) table is joined with all rows produced by the corresponding call of the table function.
-
User-defined table functions (UDTFs) must be registered before. See the UDF documentation for details on how to specify and register UDTFs.
A Temporal table function provides access to the state of a temporal table at a specific point in time.
- The syntax to join a table with a temporal table function is the same as in Join with Table Function.
Temporal Tables are tables that track changes over time.
- A Temporal Table provides access to the versions of a temporal table at a specific point in time.
{% highlight sql %}
SELECT
o.amout, o.currency, r.rate, o.amount * r.rate
@@ -666,8 +673,8 @@ FROM
JOIN LatestRates FOR SYSTEM_TIME AS OF o.proctime AS r
ON r.currency = o.currency
{% endhighlight %}
-
For more information please check the more detailed Temporal Tables concept description.
@@ -677,21 +684,21 @@ FROM
{% top %}
-#### Set Operations
+### 集合操作
-
Operation
-
Description
+
操作符
+
描述
-
+
Union
- Batch
+ 批处理
{% highlight sql %}
@@ -707,7 +714,7 @@ FROM (
UnionAll
- BatchStreaming
+ 批处理流处理
{% highlight sql %}
@@ -724,7 +731,7 @@ FROM (
Intersect / Except
- Batch
+ 批处理
{% highlight sql %}
@@ -749,10 +756,10 @@ FROM (
In
- BatchStreaming
+ 批处理流处理
-
Returns true if an expression exists in a given table sub-query. The sub-query table must consist of one column. This column must have the same data type as the expression.
{% highlight sql %}
SELECT user, amount
FROM Orders
@@ -760,17 +767,17 @@ WHERE product IN (
SELECT product FROM NewProducts
)
{% endhighlight %}
-
Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
注意： 在流查询中，这一操作将会被重写为 join 和 group 操作。该查询所需要的状态可能会由于不同的输入行数而导致无限增长。请在查询配置中提合理的保留间隔以避免产生状态过大。请阅读 查询配置 以了解详细信息
Exists
- BatchStreaming
+ 批处理流处理
-
Returns true if the sub-query returns at least one row. Only supported if the operation can be rewritten in a join and group operation.
+
若子查询的结果多于一行，将返回 true 。仅支持可以被通过 join 和 group 重写的操作。
{% highlight sql %}
SELECT user, amount
FROM Orders
@@ -778,7 +785,7 @@ WHERE product EXISTS (
SELECT product FROM NewProducts
)
{% endhighlight %}
-
Note: For streaming queries the operation is rewritten in a join and group operation. The required state to compute the query result might grow infinitely depending on the number of distinct input rows. Please provide a query configuration with valid retention interval to prevent excessive state size. See Query Configuration for details.
+
注意： 在流查询中，这一操作将会被重写为 join 和 group 操作。该查询所需要的状态可能会由于不同的输入行数而导致无限增长。请在查询配置中提合理的保留间隔以避免产生状态过大。请阅读 查询配置 以了解详细信息
@@ -787,24 +794,24 @@ WHERE product EXISTS (
{% top %}
-#### OrderBy & Limit
+### OrderBy & Limit
-
Operation
-
Description
+
操作符
+
描述
-
+
Order By
- BatchStreaming
+ 批处理流处理
-Note: The result of streaming queries must be primarily sorted on an ascending time attribute. Additional sorting attributes are supported.
+注意： 流处理结果需主要根据 时间属性 按照升序进行排序。支持使用其他排序属性。
{% highlight sql %}
SELECT *
@@ -816,10 +823,10 @@ ORDER BY orderTime
Limit
- Batch
+ 批处理
-Note: The LIMIT clause requires an ORDER BY clause.
+注意： LIMIT 查询需要有一个 ORDER BY 字句。
{% highlight sql %}
SELECT *
FROM Orders
@@ -835,16 +842,14 @@ LIMIT 3
{% top %}
-#### Top-N
+### Top-N
-Attention Top-N is only supported in Blink planner.
+注意 目前仅 Blink 计划器支持 Top-N 。
-Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in cases where the need is to display only the N bottom-most or the N top-
-most records from batch/streaming table on a condition. This result set can be used for further analysis.
+Top-N 查询是根据列排序找到N个最大或最小的值。最大值集和最小值集都被视为是一种 Top-N 的查询。若在批处理或流处理的表中需要显示出满足条件的 N 个最底层记录或最顶层记录， Top-N 查询将会十分有用。得到的结果集将可以进行进一步的分析。
-Flink uses the combination of a OVER window clause and a filter condition to express a Top-N query. With the power of OVER window `PARTITION BY` clause, Flink also supports per group Top-N. For example, the top five products per category that have the maximum sales in realtime. Top-N queries are supported for SQL on batch and streaming tables.
-
-The following shows the syntax of the TOP-N statement:
+Flink 使用 OVER 窗口条件和过滤条件相结合以进行 Top-N 查询。利用 OVER 窗口的 `PARTITION BY` 子句的功能，Flink 还支持逐组 Top-N 。 例如，每个类别中实时销量最高的前五种产品。批处理表和流处理表都支持基于SQL的 Top-N 查询。
+以下是 TOP-N 表达式的语法:
{% highlight sql %}
SELECT [column_list]
@@ -856,20 +861,20 @@ FROM (
WHERE rownum <= N [AND conditions]
{% endhighlight %}
-**Parameter Specification:**
+**参数说明：**
-- `ROW_NUMBER()`: Assigns an unique, sequential number to each row, starting with one, according to the ordering of rows within the partition. Currently, we only support `ROW_NUMBER` as the over window function. In the future, we will support `RANK()` and `DENSE_RANK()`.
-- `PARTITION BY col1[, col2...]`: Specifies the partition columns. Each partition will have a Top-N result.
-- `ORDER BY col1 [asc|desc][, col2 [asc|desc]...]`: Specifies the ordering columns. The ordering directions can be different on different columns.
-- `WHERE rownum <= N`: The `rownum <= N` is required for Flink to recognize this query is a Top-N query. The N represents the N smallest or largest records will be retained.
-- `[AND conditions]`: It is free to add other conditions in the where clause, but the other conditions can only be combined with `rownum <= N` using `AND` conjunction.
+- `ROW_NUMBER()`: 根据当前分区内的各行的顺序从第一行开始，依次为每一行分配一个唯一且连续的号码。目前，我们只支持 `ROW_NUMBER` 在 over 窗口函数中使用。未来将会支持 `RANK()` 和 `DENSE_RANK()`函数。
+- `PARTITION BY col1[, col2...]`: 指定分区列，每个分区都将会有一个 Top-N 结果。
+- `ORDER BY col1 [asc|desc][, col2 [asc|desc]...]`: 指定排序列，不同列的排序方向可以不一样。
+- `WHERE rownum <= N`: Flink 需要 `rownum <= N` 才能识别一个查询是否为 Top-N 查询。 其中， N 代表最大或最小的 N 条记录会被保留。
+- `[AND conditions]`: 在 where 语句中，可以随意添加其他的查询条件，但其他条件只允许通过 `AND` 与 `rownum <= N` 结合使用。
-Attention in Streaming Mode The TopN query is Result Updating. Flink SQL will sort the input data stream according to the order key, so if the top N records have been changed, the changed ones will be sent as retraction/update records to downstream.
-It is recommended to use a storage which supports updating as the sink of Top-N query. In addition, if the top N records need to be stored in external storage, the result table should have the same unique key with the Top-N query.
+流处理模式需注意 TopN 查询 可自动更新结果。 Flink SQL 会根据排序键对输入的流进行排序；若 top N 的记录发生了变化，变化的部分会以撤销、更新记录的形式发送到下游。
+推荐使用一个支持更新的存储作为 Top-N 查询的 sink 。另外，若 top N 记录需要存储到外部存储，则结果表需要拥有相同与 Top-N 查询相同的唯一键。
-The unique keys of Top-N query is the combination of partition columns and rownum column. Top-N query can also derive the unique key of upstream. Take following job as an example, say `product_id` is the unique key of the `ShopSales`, then the unique keys of the Top-N query are [`category`, `rownum`] and [`product_id`].
+Top-N 的唯一键是分区列和 rownum 列的结合，另外 Top-N 查询也可以获得上游的唯一键。以下面的任务为例，`product_id` 是 `ShopSales` 的唯一键，然后 Top-N 的唯一键是 [`category`, `rownum`] 和 [`product_id`] 。
-The following examples show how to specify SQL queries with Top-N on streaming tables. This is an example to get "the top five products per category that have the maximum sales in realtime" we mentioned above.
+下面的样例描述了如何指定带有 Top-N 的 SQL 查询。这个例子的作用是我们上面提到的“查询每个分类实时销量最大的五个产品”。
@@ -877,12 +882,12 @@ The following examples show how to specify SQL queries with Top-N on streaming t
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
-// ingest a DataStream from an external source
+// 接收来自外部数据源的 DataStream
DataStream> ds = env.addSource(...);
-// register the DataStream as table "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, "product_id, category, product_name, sales");
+// 把 DataStream 注册为表，表名是 “ShopSales”
+tableEnv.createTemporaryView("ShopSales", ds, "product_id, category, product_name, sales");
-// select top-5 products per category which have the maximum sales.
+// 选择每个分类中销量前5的产品
Table result1 = tableEnv.sqlQuery(
"SELECT * " +
"FROM (" +
@@ -898,13 +903,13 @@ Table result1 = tableEnv.sqlQuery(
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tableEnv = TableEnvironment.getTableEnvironment(env)
-// read a DataStream from an external source
+// 读取外部数据源的 DataStream
val ds: DataStream[(String, String, String, Long)] = env.addSource(...)
-// register the DataStream under the name "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
+// 注册名为 “ShopSales” 的 DataStream
+tableEnv.createTemporaryView("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
-// select top-5 products per category which have the maximum sales.
+// 选择每个分类中销量前5的产品
val result1 = tableEnv.sqlQuery(
"""
|SELECT *
@@ -918,13 +923,13 @@ val result1 = tableEnv.sqlQuery(
-##### No Ranking Output Optimization
+#### 无排名输出优化
-As described above, the `rownum` field will be written into the result table as one field of the unique key, which may lead to a lot of records being written to the result table. For example, when the record (say `product-1001`) of ranking 9 is updated and its rank is upgraded to 1, all the records from ranking 1 ~ 9 will be output to the result table as update messages. If the result table receives too many data, it will become the bottleneck of the SQL job.
+如上文所描述，`rownum` 字段会作为唯一键的其中一个字段写到结果表里面，这会导致大量的结果写出到结果表。比如，当原始结果（名为 `product-1001` ）从排序第九变化为排序第一时，排名 1-9 的所有结果都会以更新消息的形式发送到结果表。若结果表收到太多的数据，将会成为 SQL 任务的瓶颈。
-The optimization way is omitting rownum field in the outer SELECT clause of the Top-N query. This is reasonable because the number of the top N records is usually not large, thus the consumers can sort the records themselves quickly. Without rownum field, in the example above, only the changed record (`product-1001`) needs to be sent to downstream, which can reduce much IO to the result table.
+优化方法是在 Top-N 查询的外部 SELECT 子句中省略 rownum 字段。由于前N条记录的数量通常不大，因此消费者可以自己对记录进行快速排序，因此这是合理的。去掉 rownum 字段后，上述的例子中，只有变化了的记录（ `product-1001` ）需要发送到下游，从而可以节省大量的对结果表的 IO 操作。
-The following example shows how to optimize the above Top-N example in this way:
+以下的例子描述了如何以这种方式优化上述的 Top-N 查询：
@@ -932,12 +937,12 @@ The following example shows how to optimize the above Top-N example in this way:
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
-// ingest a DataStream from an external source
+// 从外部数据源读取 DataStream
DataStream> ds = env.addSource(...);
-// register the DataStream as table "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, "product_id, category, product_name, sales");
+// 把 DataStream 注册为表，表名是 “ShopSales”
+tableEnv.createTemporaryView("ShopSales", ds, "product_id, category, product_name, sales");
-// select top-5 products per category which have the maximum sales.
+// 选择每个分类中销量前5的产品
Table result1 = tableEnv.sqlQuery(
"SELECT product_id, category, product_name, sales " + // omit row_num field in the output
"FROM (" +
@@ -953,13 +958,13 @@ Table result1 = tableEnv.sqlQuery(
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tableEnv = TableEnvironment.getTableEnvironment(env)
-// read a DataStream from an external source
+// 从外部数据源读取 DataStream
val ds: DataStream[(String, String, String, Long)] = env.addSource(...)
-// register the DataStream under the name "ShopSales"
-tableEnv.registerDataStream("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
+// 注册名为 “ShopSales” 的数据源
+tableEnv.createTemporaryView("ShopSales", ds, 'product_id, 'category, 'product_name, 'sales)
-// select top-5 products per category which have the maximum sales.
+// 选择每个分类中销量前5的产品
val result1 = tableEnv.sqlQuery(
"""
|SELECT product_id, category, product_name, sales -- omit row_num field in the output
@@ -973,19 +978,18 @@ val result1 = tableEnv.sqlQuery(
-Attention in Streaming Mode In order to output the above query to an external storage and have a correct result, the external storage must have the same unique key with the Top-N query. In the above example query, if the `product_id` is the unique key of the query, then the external table should also has `product_id` as the unique key.
+使用流处理模式时需注意 为了使上述查询输出可以输出到外部存储并且结果正确，外部存储需要拥有与 Top-N 查询一致的唯一键。在上述的查询例子中，若 `product_id` 是查询的唯一键，那么外部表必须要有 `product_id` 作为其唯一键。
-{% top %}
+### 去重
-#### Deduplication
+注意 仅 Blink planner 支持去重。
-Attention Deduplication is only supported in Blink planner.
+去重是指对在列的集合内重复的行进行删除，只保留第一行或最后一行数据。 在某些情况下，上游的 ETL 作业不能实现精确一次的端到端，这将可能导致在故障恢复
+时，sink 中有重复的记录。 由于重复的记录将影响下游分析作业的正确性（例如，`SUM`、`COUNT`）， 所以在进一步分析之前需要进行数据去重。
-Deduplication is removing rows that duplicate over a set of columns, keeping only the first one or the last one. In some cases, the upstream ETL jobs are not end-to-end exactly-once, this may result in there are duplicate records in the sink in case of failover. However, the duplicate records will affect the correctness of downstream analytical jobs (e.g. `SUM`, `COUNT`). So a deduplication is needed before further analysis.
+与 Top-N 查询相似，Flink 使用 `ROW_NUMBER()` 去除重复的记录。理论上来说，去重是一个特殊的 Top-N 查询，其中 N 是 1 ，记录则是以处理时间或事件事件进行排序的。
-Flink uses `ROW_NUMBER()` to remove duplicates just like the way of Top-N query. In theory, deduplication is a special case of Top-N which the N is one and order by the processing time or event time.
-
-The following shows the syntax of the Deduplication statement:
+以下代码展示了去重语句的语法：
{% highlight sql %}
SELECT [column_list]
@@ -997,14 +1001,14 @@ FROM (
WHERE rownum = 1
{% endhighlight %}
-**Parameter Specification:**
+**参数说明：**
-- `ROW_NUMBER()`: Assigns an unique, sequential number to each row, starting with one.
-- `PARTITION BY col1[, col2...]`: Specifies the partition columns, i.e. the deduplicate key.
-- `ORDER BY time_attr [asc|desc]`: Specifies the ordering column, it must be a [time attribute](streaming/time_attributes.html). Currently only support [proctime attribute](streaming/time_attributes.html#processing-time). [Rowtime atttribute](streaming/time_attributes.html#event-time) will be supported in the future. Ordering by ASC means keeping the first row, ordering by DESC means keeping the last row.
-- `WHERE rownum = 1`: The `rownum = 1` is required for Flink to recognize this query is deduplication.
+- `ROW_NUMBER()`: 从第一行开始，依次为每一行分配一个唯一且连续的号码。
+- `PARTITION BY col1[, col2...]`: 指定分区的列，例如去重的键。
+- `ORDER BY time_attr [asc|desc]`: 指定排序的列。所制定的列必须为 [时间属性]({{ site.baseurl }}/zh/dev/table/streaming/time_attributes.html)。目前仅支持 [proctime attribute]({{ site.baseurl }}/zh/dev/table/streaming/time_attributes.html#processing-time)，在未来版本中将会支持 [Rowtime atttribute]({{ site.baseurl }}/zh/dev/table/streaming/time_attributes.html#event-time) 。升序（ ASC ）排列指只保留第一行，而降序排列（ DESC ）则指保留最后一行。
+- `WHERE rownum = 1`: Flink 需要 `rownum = 1` 以确定该查询是否为去重查询。
-The following examples show how to specify SQL queries with Deduplication on streaming tables.
+以下的例子描述了如何指定 SQL 查询以在一个流计算表中进行去重操作。
@@ -1012,13 +1016,12 @@ The following examples show how to specify SQL queries with Deduplication on str
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
-// ingest a DataStream from an external source
+// 从外部数据源读取 DataStream
DataStream> ds = env.addSource(...);
-// register the DataStream as table "Orders"
-tableEnv.registerDataStream("Orders", ds, "order_id, user, product, number, proctime.proctime");
+// 注册名为 “Orders” 的 DataStream
+tableEnv.createTemporaryView("Orders", ds, "order_id, user, product, number, proctime.proctime");
-// remove duplicate rows on order_id and keep the first occurrence row,
-// because there shouldn't be two orders with the same order_id.
+// 由于不应该出现两个订单有同一个order_id，所以根据 order_id 去除重复的行，并保留第一行
Table result1 = tableEnv.sqlQuery(
"SELECT order_id, user, product, number " +
"FROM (" +
@@ -1034,13 +1037,12 @@ Table result1 = tableEnv.sqlQuery(
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tableEnv = TableEnvironment.getTableEnvironment(env)
-// read a DataStream from an external source
+// 从外部数据源读取 DataStream
val ds: DataStream[(String, String, String, Int)] = env.addSource(...)
-// register the DataStream under the name "Orders"
-tableEnv.registerDataStream("Orders", ds, 'order_id, 'user, 'product, 'number, 'proctime.proctime)
+// 注册名为 “Orders” 的 DataStream
+tableEnv.createTemporaryView("Orders", ds, 'order_id, 'user, 'product, 'number, 'proctime.proctime)
-// remove duplicate rows on order_id and keep the first occurrence row,
-// because there shouldn't be two orders with the same order_id.
+// 由于不应该出现两个订单有同一个order_id，所以根据 order_id 去除重复的行，并保留第一行
val result1 = tableEnv.sqlQuery(
"""
|SELECT order_id, user, product, number
@@ -1056,82 +1058,49 @@ val result1 = tableEnv.sqlQuery(
{% top %}
-#### Insert
-
-
-
-
-
-
Operation
-
Description
-
-
-
-
-
- Insert Into
- BatchStreaming
-
-
-
Output tables must be registered in the TableEnvironment (see Register a TableSink). Moreover, the schema of the registered table must match the schema of the query.
-
-{% highlight sql %}
-INSERT INTO OutputTable
-SELECT users, tag
-FROM Orders
-{% endhighlight %}
-
-
-
-
-
-
-
-{% top %}
-
-#### Group Windows
+### 分组窗口
-Group windows are defined in the `GROUP BY` clause of a SQL query. Just like queries with regular `GROUP BY` clauses, queries with a `GROUP BY` clause that includes a group window function compute a single result row per group. The following group windows functions are supported for SQL on batch and streaming tables.
+SQL 查询的分组窗口是通过 `GROUP BY` 子句定义的。类似于使用常规 `GROUP BY` 语句的查询，窗口分组语句的 `GROUP BY` 子句中带有一个窗口函数为每个分组计算出一个结果。以下是批处理表和流处理表支持的分组窗口函数：
-
Group Window Function
-
Description
+
分组窗口函数
+
描述
TUMBLE(time_attr, interval)
-
Defines a tumbling time window. A tumbling time window assigns rows to non-overlapping, continuous windows with a fixed duration (interval). For example, a tumbling window of 5 minutes groups rows in 5 minutes intervals. Tumbling windows can be defined on event-time (stream + batch) or processing-time (stream).
Defines a hopping time window (called sliding window in the Table API). A hopping time window has a fixed duration (second interval parameter) and hops by a specified hop interval (first interval parameter). If the hop interval is smaller than the window size, hopping windows are overlapping. Thus, rows can be assigned to multiple windows. For example, a hopping window of 15 minutes size and 5 minute hop interval assigns each row to 3 different windows of 15 minute size, which are evaluated in an interval of 5 minutes. Hopping windows can be defined on event-time (stream + batch) or processing-time (stream).
Defines a session time window. Session time windows do not have a fixed duration but their bounds are defined by a time interval of inactivity, i.e., a session window is closed if no event appears for a defined gap period. For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time (stream + batch) or processing-time (stream).
-##### Time Attributes
+#### 时间属性
-For SQL queries on streaming tables, the `time_attr` argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows. See the [documentation of time attributes](streaming/time_attributes.html) to learn how to define time attributes.
+在流处理表中的 SQL 查询中，分组窗口函数的 `time_attr` 参数必须引用一个合法的时间属性，且该属性需要指定行的处理时间或事件时间。可参考 [时间属性文档]({{ site.baseurl }}/zh/dev/table/streaming/time_attributes.html) 以了解如何定义时间属性。
-For SQL on batch tables, the `time_attr` argument of the group window function must be an attribute of type `TIMESTAMP`.
+对于批处理的 SQL 查询，分组窗口函数的 `time_attr` 参数必须是一个 `TIMESTAMP` 类型的属性。
-##### Selecting Group Window Start and End Timestamps
+#### 选择分组窗口的开始和结束时间戳
-The start and end timestamps of group windows as well as time attributes can be selected with the following auxiliary functions:
+可以使用以下辅助函数选择组窗口的开始和结束时间戳以及时间属性：
-
Auxiliary Function
-
Description
+
辅助函数
+
描述
@@ -1142,7 +1111,7 @@ The start and end timestamps of group windows as well as time attributes can be
HOP_START(time_attr, interval, interval) SESSION_START(time_attr, interval)
-
Returns the timestamp of the inclusive lower bound of the corresponding tumbling, hopping, or session window.
+
返回相对应的滚动、滑动和会话窗口范围内的下界时间戳。
@@ -1150,8 +1119,8 @@ The start and end timestamps of group windows as well as time attributes can be
HOP_END(time_attr, interval, interval) SESSION_END(time_attr, interval)
-
Returns the timestamp of the exclusive upper bound of the corresponding tumbling, hopping, or session window.
@@ -1159,8 +1128,8 @@ The start and end timestamps of group windows as well as time attributes can be
HOP_ROWTIME(time_attr, interval, interval) SESSION_ROWTIME(time_attr, interval)
-
Returns the timestamp of the inclusive upper bound of the corresponding tumbling, hopping, or session window.
@@ -1168,14 +1137,14 @@ The start and end timestamps of group windows as well as time attributes can be
HOP_PROCTIME(time_attr, interval, interval) SESSION_PROCTIME(time_attr, interval)
-*Note:* Auxiliary functions must be called with exactly same arguments as the group window function in the `GROUP BY` clause.
+*注意：* 辅助函数必须使用与 `GROUP BY` 子句中的分组窗口函数完全相同的参数来调用.
-The following examples show how to specify SQL queries with group windows on streaming tables.
+以下的例子展示了如何在流处理表中指定使用分组窗口函数的 SQL 查询。
@@ -1183,27 +1152,27 @@ The following examples show how to specify SQL queries with group windows on str
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
-// ingest a DataStream from an external source
+// 从外部数据源读取 DataSource
DataStream> ds = env.addSource(...);
-// register the DataStream as table "Orders"
-tableEnv.registerDataStream("Orders", ds, "user, product, amount, proctime.proctime, rowtime.rowtime");
+// 使用“Orders”作为表名把 DataStream 注册为表
+tableEnv.createTemporaryView("Orders", ds, "user, product, amount, proctime.proctime, rowtime.rowtime");
-// compute SUM(amount) per day (in event-time)
+// 计算每日的 SUM(amount)（使用事件时间）
Table result1 = tableEnv.sqlQuery(
"SELECT user, " +
" TUMBLE_START(rowtime, INTERVAL '1' DAY) as wStart, " +
" SUM(amount) FROM Orders " +
"GROUP BY TUMBLE(rowtime, INTERVAL '1' DAY), user");
-// compute SUM(amount) per day (in processing-time)
+// 计算每日的 SUM(amount)（使用处理时间）
Table result2 = tableEnv.sqlQuery(
"SELECT user, SUM(amount) FROM Orders GROUP BY TUMBLE(proctime, INTERVAL '1' DAY), user");
-// compute every hour the SUM(amount) of the last 24 hours in event-time
+// 使用事件时间计算过去24小时中每小时的 SUM(amount)
Table result3 = tableEnv.sqlQuery(
"SELECT product, SUM(amount) FROM Orders GROUP BY HOP(rowtime, INTERVAL '1' HOUR, INTERVAL '1' DAY), product");
-// compute SUM(amount) per session with 12 hour inactivity gap (in event-time)
+// 计算每个以12小时（事件时间）作为不活动时间的会话的 SUM(amount)
Table result4 = tableEnv.sqlQuery(
"SELECT user, " +
" SESSION_START(rowtime, INTERVAL '12' HOUR) AS sStart, " +
@@ -1220,12 +1189,12 @@ Table result4 = tableEnv.sqlQuery(
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tableEnv = StreamTableEnvironment.create(env)
-// read a DataStream from an external source
+// 从外部数据源读取 DataSource
val ds: DataStream[(Long, String, Int)] = env.addSource(...)
-// register the DataStream under the name "Orders"
-tableEnv.registerDataStream("Orders", ds, 'user, 'product, 'amount, 'proctime.proctime, 'rowtime.rowtime)
+// 计算每日（使用处理时间）的 SUM(amount)
+tableEnv.createTemporaryView("Orders", ds, 'user, 'product, 'amount, 'proctime.proctime, 'rowtime.rowtime)
-// compute SUM(amount) per day (in event-time)
+// 计算每日的 SUM(amount) （使用事件时间）
val result1 = tableEnv.sqlQuery(
"""
|SELECT
@@ -1236,15 +1205,15 @@ val result1 = tableEnv.sqlQuery(
| GROUP BY TUMBLE(rowtime, INTERVAL '1' DAY), user
""".stripMargin)
-// compute SUM(amount) per day (in processing-time)
+// 计算每日的 SUM(amount) （使用处理时间）
val result2 = tableEnv.sqlQuery(
"SELECT user, SUM(amount) FROM Orders GROUP BY TUMBLE(proctime, INTERVAL '1' DAY), user")
-// compute every hour the SUM(amount) of the last 24 hours in event-time
+// 使用事件时间计算过去24小时中每小时的 SUM(amount)
val result3 = tableEnv.sqlQuery(
"SELECT product, SUM(amount) FROM Orders GROUP BY HOP(rowtime, INTERVAL '1' HOUR, INTERVAL '1' DAY), product")
-// compute SUM(amount) per session with 12 hour inactivity gap (in event-time)
+// 计算每个以12小时（事件时间）作为不活动时间的会话的 SUM(amount)
val result4 = tableEnv.sqlQuery(
"""
|SELECT
@@ -1262,25 +1231,25 @@ val result4 = tableEnv.sqlQuery(
{% top %}
-#### Pattern Recognition
+### 模式匹配
-
Operation
-
Description
+
操作符
+
描述
MATCH_RECOGNIZE
- Streaming
+ 流处理
-
Searches for a given pattern in a streaming table according to the MATCH_RECOGNIZEISO standard. This makes it possible to express complex event processing (CEP) logic in SQL queries.
{% highlight sql %}
SELECT T.aid, T.bid, T.cid
@@ -1307,144 +1276,3 @@ MATCH_RECOGNIZE (
{% top %}
-
-## DDL
-
-DDLs are specified with the `sqlUpdate()` method of the `TableEnvironment`. The method returns nothing for a success table creation. A `Table` can be register into the [Catalog](catalogs.html) with a `CREATE TABLE` statement, then can be referenced in SQL queries in method `sqlQuery()` of `TableEnvironment`.
-
-**Note:** Flink's DDL support is not yet feature complete. Queries that include unsupported SQL features cause a `TableException`. The supported features of SQL DDL on batch and streaming tables are listed in the following sections.
-
-### Specifying a DDL
-
-The following examples show how to specify a SQL DDL.
-
-
-
-{% highlight java %}
-StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
-
-// SQL query with a registered table
-// register a table named "Orders"
-tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product VARCHAR, amount INT) WITH (...)");
-// run a SQL query on the Table and retrieve the result as a new Table
-Table result = tableEnv.sqlQuery(
- "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-
-// SQL update with a registered table
-// register a TableSink
-tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product VARCHAR, amount INT) WITH (...)");
-// run a SQL update query on the Table and emit the result to the TableSink
-tableEnv.sqlUpdate(
- "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-{% endhighlight %}
-
-
-
-{% highlight scala %}
-val env = StreamExecutionEnvironment.getExecutionEnvironment
-val tableEnv = StreamTableEnvironment.create(env)
-
-// SQL query with a registered table
-// register a table named "Orders"
-tableEnv.sqlUpdate("CREATE TABLE Orders (`user` BIGINT, product VARCHAR, amount INT) WITH (...)");
-// run a SQL query on the Table and retrieve the result as a new Table
-val result = tableEnv.sqlQuery(
- "SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'");
-
-// SQL update with a registered table
-// register a TableSink
-tableEnv.sqlUpdate("CREATE TABLE RubberOrders(product VARCHAR, amount INT) WITH ('connector.path'='/path/to/file' ...)");
-// run a SQL update query on the Table and emit the result to the TableSink
-tableEnv.sqlUpdate(
- "INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
-{% endhighlight %}
-
-
-
-{% highlight python %}
-env = StreamExecutionEnvironment.get_execution_environment()
-table_env = StreamTableEnvironment.create(env)
-
-# SQL update with a registered table
-# register a TableSink
-table_env.sql_update("CREATE TABLE RubberOrders(product VARCHAR, amount INT) with (...)")
-# run a SQL update query on the Table and emit the result to the TableSink
-table_env \
- .sql_update("INSERT INTO RubberOrders SELECT product, amount FROM Orders WHERE product LIKE '%Rubber%'")
-{% endhighlight %}
-
-
-
-{% top %}
-
-### Create Table
-
-{% highlight sql %}
-CREATE TABLE [catalog_name.][db_name.]table_name
- [(col_name1 col_type1 [COMMENT col_comment1], ...)]
- [COMMENT table_comment]
- [PARTITIONED BY (col_name1, col_name2, ...)]
- WITH (key1=val1, key2=val2, ...)
-{% endhighlight %}
-
-Create a table with the given table properties. If a table with the same name already exists in the database, an exception is thrown.
-
-**PARTITIONED BY**
-
-Partition the created table by the specified columns. A directory is created for each partition if this table is used as a filesystem sink.
-
-**WITH OPTIONS**
-
-Table properties used to create a table source/sink. The properties are usually used to find and create the underlying connector.
-
-The key and value of expression `key1=val1` should both be string literal. See details in [Connect to External Systems](connect.html) for all the supported table properties of different connectors.
-
-**Notes:** The table name can be of three formats: 1. `catalog_name.db_name.table_name` 2. `db_name.table_name` 3. `table_name`. For `catalog_name.db_name.table_name`, the table would be registered into metastore with catalog named "catalog_name" and database named "db_name"; for `db_name.table_name`, the table would be registered into the current catalog of the execution table environment and database named "db_name"; for `table_name`, the table would be registered into the current catalog and database of the execution table environment.
-
-**Notes:** The table registered with `CREATE TABLE` statement can be used as both table source and table sink, we can not decide if it is used as a source or sink until it is referenced in the DMLs.
-
-{% top %}
-
-### Drop Table
-
-{% highlight sql %}
-DROP TABLE [IF EXISTS] [catalog_name.][db_name.]table_name
-{% endhighlight %}
-
-Drop a table with the given table name. If the table to drop does not exist, an exception is thrown.
-
-**IF EXISTS**
-
-If the table does not exist, nothing happens.
-
-{% top %}
-
-## Data Types
-
-Please see the dedicated page about [data types](types.html).
-
-Generic types and (nested) composite types (e.g., POJOs, tuples, rows, Scala case classes) can be fields of a row as well.
-
-Fields of composite types with arbitrary nesting can be accessed with [value access functions](functions.html#value-access-functions).
-
-Generic types are treated as a black box and can be passed on or processed by [user-defined functions](udfs.html).
-
-For DDLs, we support full data types defined in page [Data Types]({{ site.baseurl }}/dev/table/types.html).
-
-**Notes:** Some of the data types are not supported in the sql query(the cast expression or literals). E.G. `STRING`, `BYTES`, `TIME(p) WITHOUT TIME ZONE`, `TIME(p) WITH LOCAL TIME ZONE`, `TIMESTAMP(p) WITHOUT TIME ZONE`, `TIMESTAMP(p) WITH LOCAL TIME ZONE`, `ARRAY`, `MULTISET`, `ROW`.
-
-{% top %}
-
-## Reserved Keywords
-
-Although not every SQL feature is implemented yet, some string combinations are already reserved as keywords for future use. If you want to use one of the following strings as a field name, make sure to surround them with backticks (e.g. `` `value` ``, `` `count` ``).
-
-{% highlight sql %}
-
-A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, BYTES, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRING, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE
-
-{% endhighlight %}
-
-{% top %}
-
diff --git a/docs/dev/table/sqlClient.md b/docs/dev/table/sqlClient.md
index 24c63cb62d5bc0f700664b7cf3c02101e25b4294..d35a054b16571d431ba339f02a684d07ddea66fa 100644
--- a/docs/dev/table/sqlClient.md
+++ b/docs/dev/table/sqlClient.md
@@ -1,7 +1,7 @@
---
title: "SQL Client"
nav-parent_id: tableapi
-nav-pos: 100
+nav-pos: 90
is_beta: true
---
-Flink's [Table API](../tableApi.html) and [SQL support](../sql.html) are unified APIs for batch and stream processing.
+Flink's [Table API]({{ site.baseurl }}/dev/table/tableApi.html) and [SQL support]({{ site.baseurl }}/dev/table/sql/index.html) are unified APIs for batch and stream processing.
This means that Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input.
Because the relational algebra and SQL were originally designed for batch processing,
relational queries on unbounded streaming input are not as well understood as relational queries on bounded batch input.
diff --git a/docs/dev/table/streaming/index.zh.md b/docs/dev/table/streaming/index.zh.md
index a6fa525a3dbbc8e96343a6fde98a9b71ae640485..423d775926a25273a8022dca27a84f7775b15ff3 100644
--- a/docs/dev/table/streaming/index.zh.md
+++ b/docs/dev/table/streaming/index.zh.md
@@ -1,5 +1,5 @@
---
-title: "Streaming Concepts"
+title: "流式概念"
nav-id: streaming_tableapi
nav-parent_id: tableapi
nav-pos: 10
@@ -25,20 +25,20 @@ specific language governing permissions and limitations
under the License.
-->
-Flink's [Table API](../tableApi.html) and [SQL support](../sql.html) are unified APIs for batch and stream processing.
-This means that Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input.
-Because the relational algebra and SQL were originally designed for batch processing,
-relational queries on unbounded streaming input are not as well understood as relational queries on bounded batch input.
+Flink 的 [Table API]({{ site.baseurl }}/zh/dev/table/tableApi.html) 和 [SQL]({{ site.baseurl }}/zh/dev/table/sql/index.html) 是流批统一的 API。
+这意味着 Table API & SQL 在无论有限的批式输入还是无限的流式输入下，都具有相同的语义。
+因为传统的关系代数以及 SQL 最开始都是为了批式处理而设计的，
+关系型查询在流式场景下不如在批式场景下容易懂。
-The following pages explain concepts, practical limitations, and stream-specific configuration parameters of Flink's relational APIs on streaming data.
+下面这些页面包含了概念、实际的限制，以及流式数据处理中的一些特定的配置。
-Where to go next?
+接下来？
-----------------
-* [Dynamic Tables]({{ site.baseurl }}/dev/table/streaming/dynamic_tables.html): Describes the concept of dynamic tables.
-* [Time attributes]({{ site.baseurl }}/dev/table/streaming/time_attributes.html): Explains time attributes and how time attributes are handled in Table API & SQL.
-* [Joins in Continuous Queries]({{ site.baseurl }}/dev/table/streaming/joins.html): Different supported types of Joins in Continuous Queries.
-* [Temporal Tables]({{ site.baseurl }}/dev/table/streaming/temporal_tables.html): Describes the Temporal Table concept.
-* [Query configuration]({{ site.baseurl }}/dev/table/streaming/query_configuration.html): Lists Table API & SQL specific configuration options.
+* [动态表]({{ site.baseurl }}/zh/dev/table/streaming/dynamic_tables.html): 描述了动态表的概念。
+* [时间属性]({{ site.baseurl }}/zh/dev/table/streaming/time_attributes.html): 解释了时间属性以及它是如何在 Table API & SQL 中使用的。
+* [流上的 Join]({{ site.baseurl }}/zh/dev/table/streaming/joins.html): 支持的几种流上的 Join。
+* [时态（temporal）表]({{ site.baseurl }}/zh/dev/table/streaming/temporal_tables.html): 描述了时态表的概念。
+* [查询配置]({{ site.baseurl }}/zh/dev/table/streaming/query_configuration.html): Table API & SQL 特定的配置。
{% top %}
diff --git a/docs/dev/table/streaming/joins.md b/docs/dev/table/streaming/joins.md
index 476e1215cf58ee032f552172b7d4872c0ca011d5..deb31995de9ce8de2b2c47fed95e6fa12d38b0cf 100644
--- a/docs/dev/table/streaming/joins.md
+++ b/docs/dev/table/streaming/joins.md
@@ -26,7 +26,7 @@ Joins are a common and well-understood operation in batch data processing to con
Because of that, there are a couple of ways to actually perform a join using either Table API or SQL.
-For more information regarding the syntax, please check the join sections in [Table API](../tableApi.html#joins) and [SQL](../sql.html#joins).
+For more information regarding the syntax, please check the join sections in [Table API](../tableApi.html#joins) and [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#joins).
* This will be replaced by the TOC
{:toc}
diff --git a/docs/dev/table/streaming/joins.zh.md b/docs/dev/table/streaming/joins.zh.md
index 476e1215cf58ee032f552172b7d4872c0ca011d5..deb31995de9ce8de2b2c47fed95e6fa12d38b0cf 100644
--- a/docs/dev/table/streaming/joins.zh.md
+++ b/docs/dev/table/streaming/joins.zh.md
@@ -26,7 +26,7 @@ Joins are a common and well-understood operation in batch data processing to con
Because of that, there are a couple of ways to actually perform a join using either Table API or SQL.
-For more information regarding the syntax, please check the join sections in [Table API](../tableApi.html#joins) and [SQL](../sql.html#joins).
+For more information regarding the syntax, please check the join sections in [Table API](../tableApi.html#joins) and [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#joins).
* This will be replaced by the TOC
{:toc}
diff --git a/docs/dev/table/streaming/match_recognize.md b/docs/dev/table/streaming/match_recognize.md
index 663c7d2a59132345b7a5776b11fd3d68fdfcfc23..8fb12fa5494c90272f5d50959aae9ffe068769a7 100644
--- a/docs/dev/table/streaming/match_recognize.md
+++ b/docs/dev/table/streaming/match_recognize.md
@@ -263,8 +263,8 @@ look at the [event stream navigation](#pattern-navigation) section.
### Aggregations
Aggregations can be used in `DEFINE` and `MEASURES` clauses. Both
-[built-in]({{ site.baseurl }}/dev/table/sql.html#built-in-functions) and custom
-[user defined]({{ site.baseurl }}/dev/table/udfs.html) functions are supported.
+[built-in]({{ site.baseurl }}/dev/table/functions/systemFunctions.html) and custom
+[user defined]({{ site.baseurl }}/dev/table/functions/udfs.html) functions are supported.
Aggregate functions are applied to each subset of rows mapped to a match. In order to understand
how those subsets are evaluated have a look at the [event stream navigation](#pattern-navigation)
@@ -715,7 +715,7 @@ variable. This can be expressed with two corresponding functions:
- {% highlight text %}
+{% highlight text %}
LAST(variable.field, n)
{% endhighlight %}
@@ -726,7 +726,7 @@ LAST(variable.field, n)
- {% highlight text %}
+{% highlight text %}
FIRST(variable.field, n)
{% endhighlight %}
diff --git a/docs/dev/table/streaming/match_recognize.zh.md b/docs/dev/table/streaming/match_recognize.zh.md
index 663c7d2a59132345b7a5776b11fd3d68fdfcfc23..8fb12fa5494c90272f5d50959aae9ffe068769a7 100644
--- a/docs/dev/table/streaming/match_recognize.zh.md
+++ b/docs/dev/table/streaming/match_recognize.zh.md
@@ -263,8 +263,8 @@ look at the [event stream navigation](#pattern-navigation) section.
### Aggregations
Aggregations can be used in `DEFINE` and `MEASURES` clauses. Both
-[built-in]({{ site.baseurl }}/dev/table/sql.html#built-in-functions) and custom
-[user defined]({{ site.baseurl }}/dev/table/udfs.html) functions are supported.
+[built-in]({{ site.baseurl }}/dev/table/functions/systemFunctions.html) and custom
+[user defined]({{ site.baseurl }}/dev/table/functions/udfs.html) functions are supported.
Aggregate functions are applied to each subset of rows mapped to a match. In order to understand
how those subsets are evaluated have a look at the [event stream navigation](#pattern-navigation)
@@ -715,7 +715,7 @@ variable. This can be expressed with two corresponding functions:
- {% highlight text %}
+{% highlight text %}
LAST(variable.field, n)
{% endhighlight %}
@@ -726,7 +726,7 @@ LAST(variable.field, n)
- {% highlight text %}
+{% highlight text %}
FIRST(variable.field, n)
{% endhighlight %}
diff --git a/docs/dev/table/streaming/query_configuration.md b/docs/dev/table/streaming/query_configuration.md
index dbb18bc850c5dcb38ec88ccfc220e5155ec720d3..2a2f50cead1912fe9246ec4909fab3b60835573b 100644
--- a/docs/dev/table/streaming/query_configuration.md
+++ b/docs/dev/table/streaming/query_configuration.md
@@ -24,7 +24,7 @@ under the License.
Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input. In many cases, continuous queries on streaming input are capable of computing accurate results that are identical to offline computed results. However, this is not possible in general case because continuous queries have to restrict the size of the state they are maintaining in order to avoid to run out of storage and to be able to process unbounded streaming data over a long period of time. As a result, a continuous query might only be able to provide approximated results depending on the characteristics of the input data and the query itself.
-Flink's Table API and SQL interface provide parameters to tune the accuracy and resource consumption of continuous queries. The parameters are specified via a `QueryConfig` object. The `QueryConfig` can be obtained from the `TableEnvironment` and is passed back when a `Table` is translated, i.e., when it is [transformed into a DataStream](../common.html#convert-a-table-into-a-datastream-or-dataset) or [emitted via a TableSink](../common.html#emit-a-table).
+Flink's Table API and SQL interface provide parameters to tune the accuracy and resource consumption of continuous queries. The parameters are specified via a `QueryConfig` object. The `QueryConfig` can be obtained from the `TableEnvironment` and is passed back when a `Table` is translated, i.e., when it is [transformed into a DataStream]({{ site.baseurl }}/dev/table/common.html#convert-a-table-into-a-datastream-or-dataset) or [emitted via a TableSink](../common.html#emit-a-table).
diff --git a/docs/dev/table/streaming/query_configuration.zh.md b/docs/dev/table/streaming/query_configuration.zh.md
index dbb18bc850c5dcb38ec88ccfc220e5155ec720d3..2a2f50cead1912fe9246ec4909fab3b60835573b 100644
--- a/docs/dev/table/streaming/query_configuration.zh.md
+++ b/docs/dev/table/streaming/query_configuration.zh.md
@@ -24,7 +24,7 @@ under the License.
Table API and SQL queries have the same semantics regardless whether their input is bounded batch input or unbounded stream input. In many cases, continuous queries on streaming input are capable of computing accurate results that are identical to offline computed results. However, this is not possible in general case because continuous queries have to restrict the size of the state they are maintaining in order to avoid to run out of storage and to be able to process unbounded streaming data over a long period of time. As a result, a continuous query might only be able to provide approximated results depending on the characteristics of the input data and the query itself.
-Flink's Table API and SQL interface provide parameters to tune the accuracy and resource consumption of continuous queries. The parameters are specified via a `QueryConfig` object. The `QueryConfig` can be obtained from the `TableEnvironment` and is passed back when a `Table` is translated, i.e., when it is [transformed into a DataStream](../common.html#convert-a-table-into-a-datastream-or-dataset) or [emitted via a TableSink](../common.html#emit-a-table).
+Flink's Table API and SQL interface provide parameters to tune the accuracy and resource consumption of continuous queries. The parameters are specified via a `QueryConfig` object. The `QueryConfig` can be obtained from the `TableEnvironment` and is passed back when a `Table` is translated, i.e., when it is [transformed into a DataStream]({{ site.baseurl }}/dev/table/common.html#convert-a-table-into-a-datastream-or-dataset) or [emitted via a TableSink](../common.html#emit-a-table).
diff --git a/docs/dev/table/streaming/temporal_tables.md b/docs/dev/table/streaming/temporal_tables.md
index baecbd07d5533b458c7b50b6821340202ff2f599..00c919209e01c0618b5447594110716d0fd432bc 100644
--- a/docs/dev/table/streaming/temporal_tables.md
+++ b/docs/dev/table/streaming/temporal_tables.md
@@ -119,7 +119,7 @@ Temporal Table Function
------------------------
In order to access the data in a temporal table, one must pass a [time attribute](time_attributes.html) that determines the version of the table that will be returned.
-Flink uses the SQL syntax of [table functions](../udfs.html#table-functions) to provide a way to express it.
+Flink uses the SQL syntax of [table functions]({{ site.baseurl }}/dev/table/functions/udfs.html#table-functions) to provide a way to express it.
Once defined, a *Temporal Table Function* takes a single time argument `timeAttribute` and returns a set of rows.
This set contains the latest versions of the rows for all of the existing primary keys with respect to the given time attribute.
@@ -178,7 +178,7 @@ ratesHistoryData.add(Tuple2.of("Euro", 119L));
DataStream> ratesHistoryStream = env.fromCollection(ratesHistoryData);
Table ratesHistory = tEnv.fromDataStream(ratesHistoryStream, "r_currency, r_rate, r_proctime.proctime");
-tEnv.registerTable("RatesHistory", ratesHistory);
+tEnv.createTemporaryView("RatesHistory", ratesHistory);
// Create and register a temporal table function.
// Define "r_proctime" as the time attribute and "r_currency" as the primary key.
@@ -206,7 +206,7 @@ val ratesHistory = env
.fromCollection(ratesHistoryData)
.toTable(tEnv, 'r_currency, 'r_rate, 'r_proctime.proctime)
-tEnv.registerTable("RatesHistory", ratesHistory)
+tEnv.createTemporaryView("RatesHistory", ratesHistory)
// Create and register TemporalTableFunction.
// Define "r_proctime" as the time attribute and "r_currency" as the primary key.
@@ -220,7 +220,7 @@ Line `(1)` creates a `rates` [temporal table function](#temporal-table-functions
which allows us to use the function `rates` in the [Table API](../tableApi.html#joins).
Line `(2)` registers this function under the name `Rates` in our table environment,
-which allows us to use the `Rates` function in [SQL](../sql.html#joins).
+which allows us to use the `Rates` function in [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#joins).
## Temporal Table
diff --git a/docs/dev/table/streaming/temporal_tables.zh.md b/docs/dev/table/streaming/temporal_tables.zh.md
index baecbd07d5533b458c7b50b6821340202ff2f599..00c919209e01c0618b5447594110716d0fd432bc 100644
--- a/docs/dev/table/streaming/temporal_tables.zh.md
+++ b/docs/dev/table/streaming/temporal_tables.zh.md
@@ -119,7 +119,7 @@ Temporal Table Function
------------------------
In order to access the data in a temporal table, one must pass a [time attribute](time_attributes.html) that determines the version of the table that will be returned.
-Flink uses the SQL syntax of [table functions](../udfs.html#table-functions) to provide a way to express it.
+Flink uses the SQL syntax of [table functions]({{ site.baseurl }}/dev/table/functions/udfs.html#table-functions) to provide a way to express it.
Once defined, a *Temporal Table Function* takes a single time argument `timeAttribute` and returns a set of rows.
This set contains the latest versions of the rows for all of the existing primary keys with respect to the given time attribute.
@@ -178,7 +178,7 @@ ratesHistoryData.add(Tuple2.of("Euro", 119L));
DataStream> ratesHistoryStream = env.fromCollection(ratesHistoryData);
Table ratesHistory = tEnv.fromDataStream(ratesHistoryStream, "r_currency, r_rate, r_proctime.proctime");
-tEnv.registerTable("RatesHistory", ratesHistory);
+tEnv.createTemporaryView("RatesHistory", ratesHistory);
// Create and register a temporal table function.
// Define "r_proctime" as the time attribute and "r_currency" as the primary key.
@@ -206,7 +206,7 @@ val ratesHistory = env
.fromCollection(ratesHistoryData)
.toTable(tEnv, 'r_currency, 'r_rate, 'r_proctime.proctime)
-tEnv.registerTable("RatesHistory", ratesHistory)
+tEnv.createTemporaryView("RatesHistory", ratesHistory)
// Create and register TemporalTableFunction.
// Define "r_proctime" as the time attribute and "r_currency" as the primary key.
@@ -220,7 +220,7 @@ Line `(1)` creates a `rates` [temporal table function](#temporal-table-functions
which allows us to use the function `rates` in the [Table API](../tableApi.html#joins).
Line `(2)` registers this function under the name `Rates` in our table environment,
-which allows us to use the `Rates` function in [SQL](../sql.html#joins).
+which allows us to use the `Rates` function in [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#joins).
## Temporal Table
diff --git a/docs/dev/table/streaming/time_attributes.md b/docs/dev/table/streaming/time_attributes.md
index 20e4301bc25c2ed790a490f6544c96f64f05ce7e..3b9c892eeb9369fabeb7c9794a5e92733726874f 100644
--- a/docs/dev/table/streaming/time_attributes.md
+++ b/docs/dev/table/streaming/time_attributes.md
@@ -38,9 +38,9 @@ This page explains how time attributes can be defined for time-based operations
Introduction to Time Attributes
-------------------------------
-Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs.
+Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs.
-Time attributes can be part of every table schema. They are defined when creating a table from a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can be used in time-based operations.
+Time attributes can be part of every table schema. They are defined when creating a table from a CREATE TABLE DDL or a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can be used in time-based operations.
As long as a time attribute is not modified and is simply forwarded from one part of the query to another, it remains a valid time attribute. Time attributes behave like regular timestamps and can be accessed for calculations. If a time attribute is used in a calculation, it will be materialized and becomes a regular timestamp. Regular timestamps do not cooperate with Flink's time and watermarking system and thus can not be used for time-based operations anymore.
@@ -87,7 +87,28 @@ Processing time
Processing time allows a table program to produce results based on the time of the local machine. It is the simplest notion of time but does not provide determinism. It neither requires timestamp extraction nor watermark generation.
-There are two ways to define a processing time attribute.
+There are three ways to define a processing time attribute.
+
+### Defining in create table DDL
+
+The processing time attribute is defined as a computed column in create table DDL using the system `PROCTIME()` function. Please see [CREATE TABLE DDL]({{ site.baseurl }}/dev/table/sql/create.html#create-table) for more information about computed column.
+
+{% highlight sql %}
+
+CREATE TABLE user_actions (
+ user_name STRING,
+ data STRING,
+ user_action_time AS PROCTIME() -- declare an additional field as a processing time attribute
+) WITH (
+ ...
+);
+
+SELECT TUMBLE_START(user_action_time, INTERVAL '10' MINUTE), COUNT(DISTINCT user_name)
+FROM user_actions
+GROUP BY TUMBLE(user_action_time, INTERVAL '10' MINUTE);
+
+{% endhighlight %}
+
### During DataStream-to-Table Conversion
@@ -99,9 +120,9 @@ The processing time attribute is defined with the `.proctime` property during sc
DataStream> stream = ...;
// declare an additional logical field as a processing time attribute
-Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.proctime");
+Table table = tEnv.fromDataStream(stream, "user_name, data, user_action_time.proctime");
-WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -109,9 +130,9 @@ WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserAct
val stream: DataStream[(String, String)] = ...
// declare an additional logical field as a processing time attribute
-val table = tEnv.fromDataStream(stream, 'UserActionTimestamp, 'Username, 'Data, 'UserActionTime.proctime)
+val table = tEnv.fromDataStream(stream, 'UserActionTimestamp, 'user_name, 'data, 'user_action_time.proctime)
-val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+val windowedTable = table.window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
@@ -128,7 +149,7 @@ public class UserActionSource implements StreamTableSource, DefinedProctime
@Override
public TypeInformation getReturnType() {
- String[] names = new String[] {"Username" , "Data"};
+ String[] names = new String[] {"user_name" , "data"};
TypeInformation[] types = new TypeInformation[] {Types.STRING(), Types.STRING()};
return Types.ROW(names, types);
}
@@ -143,16 +164,16 @@ public class UserActionSource implements StreamTableSource, DefinedProctime
@Override
public String getProctimeAttribute() {
// field with this name will be appended as a third field
- return "UserActionTime";
+ return "user_action_time";
}
}
// register table source
-tEnv.registerTableSource("UserActions", new UserActionSource());
+tEnv.registerTableSource("user_actions", new UserActionSource());
WindowedTable windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+ .from("user_actions")
+ .window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -161,7 +182,7 @@ WindowedTable windowedTable = tEnv
class UserActionSource extends StreamTableSource[Row] with DefinedProctimeAttribute {
override def getReturnType = {
- val names = Array[String]("Username" , "Data")
+ val names = Array[String]("user_name" , "data")
val types = Array[TypeInformation[_]](Types.STRING, Types.STRING)
Types.ROW(names, types)
}
@@ -174,16 +195,16 @@ class UserActionSource extends StreamTableSource[Row] with DefinedProctimeAttrib
override def getProctimeAttribute = {
// field with this name will be appended as a third field
- "UserActionTime"
+ "user_action_time"
}
}
// register table source
-tEnv.registerTableSource("UserActions", new UserActionSource)
+tEnv.registerTableSource("user_actions", new UserActionSource)
val windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+ .from("user_actions")
+ .window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
@@ -197,7 +218,30 @@ Additionally, event time allows for unified syntax for table programs in both ba
In order to handle out-of-order events and distinguish between on-time and late events in streaming, Flink needs to extract timestamps from events and make some kind of progress in time (so-called [watermarks]({{ site.baseurl }}/dev/event_time.html)).
-An event time attribute can be defined either during DataStream-to-Table conversion or by using a TableSource.
+An event time attribute can be defined either in create table DDL or during DataStream-to-Table conversion or by using a TableSource.
+
+### Defining in create table DDL
+
+The event time attribute is defined using WATERMARK statement in CREATE TABLE DDL. A watermark statement defines a watermark generation expression on an existing event time field, which marks the event time field as event time attribute. Please see [CREATE TABLE DDL]({{ site.baseurl }}/dev/table/sql/create.html#create-table) for more information about watermark statement and watermark strategies.
+
+{% highlight sql %}
+
+CREATE TABLE user_actions (
+ user_name STRING,
+ data STRING,
+ user_action_time TIMESTAMP(3),
+ -- declare user_action_time as event time attribute and use 5 seconds delayed watermark strategy
+ WATERMARK FOR user_action_time AS user_action_time - INTERVAL '5' SECOND
+) WITH (
+ ...
+);
+
+SELECT TUMBLE_START(user_action_time, INTERVAL '10' MINUTE), COUNT(DISTINCT user_name)
+FROM user_actions
+GROUP BY TUMBLE(user_action_time, INTERVAL '10' MINUTE);
+
+{% endhighlight %}
+
### During DataStream-to-Table Conversion
@@ -220,7 +264,7 @@ In either case the event time timestamp field will hold the value of the `DataSt
DataStream> stream = inputStream.assignTimestampsAndWatermarks(...);
// declare an additional logical field as an event time attribute
-Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.rowtime");
+Table table = tEnv.fromDataStream(stream, "user_name, data, user_action_time.rowtime");
// Option 2:
@@ -230,11 +274,11 @@ DataStream> stream = inputStream.assignTimestampsAn
// the first field has been used for timestamp extraction, and is no longer necessary
// replace first field with a logical event time attribute
-Table table = tEnv.fromDataStream(stream, "UserActionTime.rowtime, Username, Data");
+Table table = tEnv.fromDataStream(stream, "user_action_time.rowtime, user_name, data");
// Usage:
-WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -246,7 +290,7 @@ WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserAct
val stream: DataStream[(String, String)] = inputStream.assignTimestampsAndWatermarks(...)
// declare an additional logical field as an event time attribute
-val table = tEnv.fromDataStream(stream, 'Username, 'Data, 'UserActionTime.rowtime)
+val table = tEnv.fromDataStream(stream, 'user_name, 'data, 'user_action_time.rowtime)
// Option 2:
@@ -256,11 +300,11 @@ val stream: DataStream[(Long, String, String)] = inputStream.assignTimestampsAnd
// the first field has been used for timestamp extraction, and is no longer necessary
// replace first field with a logical event time attribute
-val table = tEnv.fromDataStream(stream, 'UserActionTime.rowtime, 'Username, 'Data)
+val table = tEnv.fromDataStream(stream, 'user_action_time.rowtime, 'user_name, 'data)
// Usage:
-val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+val windowedTable = table.window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
@@ -282,7 +326,7 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA
@Override
public TypeInformation getReturnType() {
- String[] names = new String[] {"Username", "Data", "UserActionTime"};
+ String[] names = new String[] {"user_name", "data", "user_action_time"};
TypeInformation[] types =
new TypeInformation[] {Types.STRING(), Types.STRING(), Types.LONG()};
return Types.ROW(names, types);
@@ -292,18 +336,18 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA
public DataStream getDataStream(StreamExecutionEnvironment execEnv) {
// create stream
// ...
- // assign watermarks based on the "UserActionTime" attribute
+ // assign watermarks based on the "user_action_time" attribute
DataStream stream = inputStream.assignTimestampsAndWatermarks(...);
return stream;
}
@Override
public List getRowtimeAttributeDescriptors() {
- // Mark the "UserActionTime" attribute as event-time attribute.
- // We create one attribute descriptor of "UserActionTime".
+ // Mark the "user_action_time" attribute as event-time attribute.
+ // We create one attribute descriptor of "user_action_time".
RowtimeAttributeDescriptor rowtimeAttrDescr = new RowtimeAttributeDescriptor(
- "UserActionTime",
- new ExistingField("UserActionTime"),
+ "user_action_time",
+ new ExistingField("user_action_time"),
new AscendingTimestamps());
List listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr);
return listRowtimeAttrDescr;
@@ -311,11 +355,11 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA
}
// register the table source
-tEnv.registerTableSource("UserActions", new UserActionSource());
+tEnv.registerTableSource("user_actions", new UserActionSource());
WindowedTable windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+ .from("user_actions")
+ .window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -324,7 +368,7 @@ WindowedTable windowedTable = tEnv
class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttributes {
override def getReturnType = {
- val names = Array[String]("Username" , "Data", "UserActionTime")
+ val names = Array[String]("user_name" , "data", "user_action_time")
val types = Array[TypeInformation[_]](Types.STRING, Types.STRING, Types.LONG)
Types.ROW(names, types)
}
@@ -332,17 +376,17 @@ class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribu
override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = {
// create stream
// ...
- // assign watermarks based on the "UserActionTime" attribute
+ // assign watermarks based on the "user_action_time" attribute
val stream = inputStream.assignTimestampsAndWatermarks(...)
stream
}
override def getRowtimeAttributeDescriptors: util.List[RowtimeAttributeDescriptor] = {
- // Mark the "UserActionTime" attribute as event-time attribute.
- // We create one attribute descriptor of "UserActionTime".
+ // Mark the "user_action_time" attribute as event-time attribute.
+ // We create one attribute descriptor of "user_action_time".
val rowtimeAttrDescr = new RowtimeAttributeDescriptor(
- "UserActionTime",
- new ExistingField("UserActionTime"),
+ "user_action_time",
+ new ExistingField("user_action_time"),
new AscendingTimestamps)
val listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr)
listRowtimeAttrDescr
@@ -350,11 +394,11 @@ class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribu
}
// register the table source
-tEnv.registerTableSource("UserActions", new UserActionSource)
+tEnv.registerTableSource("user_actions", new UserActionSource)
val windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+ .from("user_actions")
+ .window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
diff --git a/docs/dev/table/streaming/time_attributes.zh.md b/docs/dev/table/streaming/time_attributes.zh.md
index 20e4301bc25c2ed790a490f6544c96f64f05ce7e..3b9c892eeb9369fabeb7c9794a5e92733726874f 100644
--- a/docs/dev/table/streaming/time_attributes.zh.md
+++ b/docs/dev/table/streaming/time_attributes.zh.md
@@ -38,9 +38,9 @@ This page explains how time attributes can be defined for time-based operations
Introduction to Time Attributes
-------------------------------
-Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs.
+Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql/queries.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs.
-Time attributes can be part of every table schema. They are defined when creating a table from a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can be used in time-based operations.
+Time attributes can be part of every table schema. They are defined when creating a table from a CREATE TABLE DDL or a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can be used in time-based operations.
As long as a time attribute is not modified and is simply forwarded from one part of the query to another, it remains a valid time attribute. Time attributes behave like regular timestamps and can be accessed for calculations. If a time attribute is used in a calculation, it will be materialized and becomes a regular timestamp. Regular timestamps do not cooperate with Flink's time and watermarking system and thus can not be used for time-based operations anymore.
@@ -87,7 +87,28 @@ Processing time
Processing time allows a table program to produce results based on the time of the local machine. It is the simplest notion of time but does not provide determinism. It neither requires timestamp extraction nor watermark generation.
-There are two ways to define a processing time attribute.
+There are three ways to define a processing time attribute.
+
+### Defining in create table DDL
+
+The processing time attribute is defined as a computed column in create table DDL using the system `PROCTIME()` function. Please see [CREATE TABLE DDL]({{ site.baseurl }}/dev/table/sql/create.html#create-table) for more information about computed column.
+
+{% highlight sql %}
+
+CREATE TABLE user_actions (
+ user_name STRING,
+ data STRING,
+ user_action_time AS PROCTIME() -- declare an additional field as a processing time attribute
+) WITH (
+ ...
+);
+
+SELECT TUMBLE_START(user_action_time, INTERVAL '10' MINUTE), COUNT(DISTINCT user_name)
+FROM user_actions
+GROUP BY TUMBLE(user_action_time, INTERVAL '10' MINUTE);
+
+{% endhighlight %}
+
### During DataStream-to-Table Conversion
@@ -99,9 +120,9 @@ The processing time attribute is defined with the `.proctime` property during sc
DataStream> stream = ...;
// declare an additional logical field as a processing time attribute
-Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.proctime");
+Table table = tEnv.fromDataStream(stream, "user_name, data, user_action_time.proctime");
-WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -109,9 +130,9 @@ WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserAct
val stream: DataStream[(String, String)] = ...
// declare an additional logical field as a processing time attribute
-val table = tEnv.fromDataStream(stream, 'UserActionTimestamp, 'Username, 'Data, 'UserActionTime.proctime)
+val table = tEnv.fromDataStream(stream, 'UserActionTimestamp, 'user_name, 'data, 'user_action_time.proctime)
-val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+val windowedTable = table.window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
@@ -128,7 +149,7 @@ public class UserActionSource implements StreamTableSource, DefinedProctime
@Override
public TypeInformation getReturnType() {
- String[] names = new String[] {"Username" , "Data"};
+ String[] names = new String[] {"user_name" , "data"};
TypeInformation[] types = new TypeInformation[] {Types.STRING(), Types.STRING()};
return Types.ROW(names, types);
}
@@ -143,16 +164,16 @@ public class UserActionSource implements StreamTableSource, DefinedProctime
@Override
public String getProctimeAttribute() {
// field with this name will be appended as a third field
- return "UserActionTime";
+ return "user_action_time";
}
}
// register table source
-tEnv.registerTableSource("UserActions", new UserActionSource());
+tEnv.registerTableSource("user_actions", new UserActionSource());
WindowedTable windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+ .from("user_actions")
+ .window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -161,7 +182,7 @@ WindowedTable windowedTable = tEnv
class UserActionSource extends StreamTableSource[Row] with DefinedProctimeAttribute {
override def getReturnType = {
- val names = Array[String]("Username" , "Data")
+ val names = Array[String]("user_name" , "data")
val types = Array[TypeInformation[_]](Types.STRING, Types.STRING)
Types.ROW(names, types)
}
@@ -174,16 +195,16 @@ class UserActionSource extends StreamTableSource[Row] with DefinedProctimeAttrib
override def getProctimeAttribute = {
// field with this name will be appended as a third field
- "UserActionTime"
+ "user_action_time"
}
}
// register table source
-tEnv.registerTableSource("UserActions", new UserActionSource)
+tEnv.registerTableSource("user_actions", new UserActionSource)
val windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+ .from("user_actions")
+ .window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
@@ -197,7 +218,30 @@ Additionally, event time allows for unified syntax for table programs in both ba
In order to handle out-of-order events and distinguish between on-time and late events in streaming, Flink needs to extract timestamps from events and make some kind of progress in time (so-called [watermarks]({{ site.baseurl }}/dev/event_time.html)).
-An event time attribute can be defined either during DataStream-to-Table conversion or by using a TableSource.
+An event time attribute can be defined either in create table DDL or during DataStream-to-Table conversion or by using a TableSource.
+
+### Defining in create table DDL
+
+The event time attribute is defined using WATERMARK statement in CREATE TABLE DDL. A watermark statement defines a watermark generation expression on an existing event time field, which marks the event time field as event time attribute. Please see [CREATE TABLE DDL]({{ site.baseurl }}/dev/table/sql/create.html#create-table) for more information about watermark statement and watermark strategies.
+
+{% highlight sql %}
+
+CREATE TABLE user_actions (
+ user_name STRING,
+ data STRING,
+ user_action_time TIMESTAMP(3),
+ -- declare user_action_time as event time attribute and use 5 seconds delayed watermark strategy
+ WATERMARK FOR user_action_time AS user_action_time - INTERVAL '5' SECOND
+) WITH (
+ ...
+);
+
+SELECT TUMBLE_START(user_action_time, INTERVAL '10' MINUTE), COUNT(DISTINCT user_name)
+FROM user_actions
+GROUP BY TUMBLE(user_action_time, INTERVAL '10' MINUTE);
+
+{% endhighlight %}
+
### During DataStream-to-Table Conversion
@@ -220,7 +264,7 @@ In either case the event time timestamp field will hold the value of the `DataSt
DataStream> stream = inputStream.assignTimestampsAndWatermarks(...);
// declare an additional logical field as an event time attribute
-Table table = tEnv.fromDataStream(stream, "Username, Data, UserActionTime.rowtime");
+Table table = tEnv.fromDataStream(stream, "user_name, data, user_action_time.rowtime");
// Option 2:
@@ -230,11 +274,11 @@ DataStream> stream = inputStream.assignTimestampsAn
// the first field has been used for timestamp extraction, and is no longer necessary
// replace first field with a logical event time attribute
-Table table = tEnv.fromDataStream(stream, "UserActionTime.rowtime, Username, Data");
+Table table = tEnv.fromDataStream(stream, "user_action_time.rowtime, user_name, data");
// Usage:
-WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -246,7 +290,7 @@ WindowedTable windowedTable = table.window(Tumble.over("10.minutes").on("UserAct
val stream: DataStream[(String, String)] = inputStream.assignTimestampsAndWatermarks(...)
// declare an additional logical field as an event time attribute
-val table = tEnv.fromDataStream(stream, 'Username, 'Data, 'UserActionTime.rowtime)
+val table = tEnv.fromDataStream(stream, 'user_name, 'data, 'user_action_time.rowtime)
// Option 2:
@@ -256,11 +300,11 @@ val stream: DataStream[(Long, String, String)] = inputStream.assignTimestampsAnd
// the first field has been used for timestamp extraction, and is no longer necessary
// replace first field with a logical event time attribute
-val table = tEnv.fromDataStream(stream, 'UserActionTime.rowtime, 'Username, 'Data)
+val table = tEnv.fromDataStream(stream, 'user_action_time.rowtime, 'user_name, 'data)
// Usage:
-val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+val windowedTable = table.window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
@@ -282,7 +326,7 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA
@Override
public TypeInformation getReturnType() {
- String[] names = new String[] {"Username", "Data", "UserActionTime"};
+ String[] names = new String[] {"user_name", "data", "user_action_time"};
TypeInformation[] types =
new TypeInformation[] {Types.STRING(), Types.STRING(), Types.LONG()};
return Types.ROW(names, types);
@@ -292,18 +336,18 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA
public DataStream getDataStream(StreamExecutionEnvironment execEnv) {
// create stream
// ...
- // assign watermarks based on the "UserActionTime" attribute
+ // assign watermarks based on the "user_action_time" attribute
DataStream stream = inputStream.assignTimestampsAndWatermarks(...);
return stream;
}
@Override
public List getRowtimeAttributeDescriptors() {
- // Mark the "UserActionTime" attribute as event-time attribute.
- // We create one attribute descriptor of "UserActionTime".
+ // Mark the "user_action_time" attribute as event-time attribute.
+ // We create one attribute descriptor of "user_action_time".
RowtimeAttributeDescriptor rowtimeAttrDescr = new RowtimeAttributeDescriptor(
- "UserActionTime",
- new ExistingField("UserActionTime"),
+ "user_action_time",
+ new ExistingField("user_action_time"),
new AscendingTimestamps());
List listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr);
return listRowtimeAttrDescr;
@@ -311,11 +355,11 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA
}
// register the table source
-tEnv.registerTableSource("UserActions", new UserActionSource());
+tEnv.registerTableSource("user_actions", new UserActionSource());
WindowedTable windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble.over("10.minutes").on("UserActionTime").as("userActionWindow"));
+ .from("user_actions")
+ .window(Tumble.over("10.minutes").on("user_action_time").as("userActionWindow"));
{% endhighlight %}
@@ -324,7 +368,7 @@ WindowedTable windowedTable = tEnv
class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttributes {
override def getReturnType = {
- val names = Array[String]("Username" , "Data", "UserActionTime")
+ val names = Array[String]("user_name" , "data", "user_action_time")
val types = Array[TypeInformation[_]](Types.STRING, Types.STRING, Types.LONG)
Types.ROW(names, types)
}
@@ -332,17 +376,17 @@ class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribu
override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = {
// create stream
// ...
- // assign watermarks based on the "UserActionTime" attribute
+ // assign watermarks based on the "user_action_time" attribute
val stream = inputStream.assignTimestampsAndWatermarks(...)
stream
}
override def getRowtimeAttributeDescriptors: util.List[RowtimeAttributeDescriptor] = {
- // Mark the "UserActionTime" attribute as event-time attribute.
- // We create one attribute descriptor of "UserActionTime".
+ // Mark the "user_action_time" attribute as event-time attribute.
+ // We create one attribute descriptor of "user_action_time".
val rowtimeAttrDescr = new RowtimeAttributeDescriptor(
- "UserActionTime",
- new ExistingField("UserActionTime"),
+ "user_action_time",
+ new ExistingField("user_action_time"),
new AscendingTimestamps)
val listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr)
listRowtimeAttrDescr
@@ -350,11 +394,11 @@ class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribu
}
// register the table source
-tEnv.registerTableSource("UserActions", new UserActionSource)
+tEnv.registerTableSource("user_actions", new UserActionSource)
val windowedTable = tEnv
- .scan("UserActions")
- .window(Tumble over 10.minutes on 'UserActionTime as 'userActionWindow)
+ .from("user_actions")
+ .window(Tumble over 10.minutes on 'user_action_time as 'userActionWindow)
{% endhighlight %}
diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md
index c677de62ab10fc8fca6bcac47007f04dab0fafde..f2b5624c22a8b3777a8236e3adcbf6e510cf00f4 100644
--- a/docs/dev/table/tableApi.md
+++ b/docs/dev/table/tableApi.md
@@ -1,7 +1,7 @@
---
title: "Table API"
nav-parent_id: tableapi
-nav-pos: 20
+nav-pos: 30
---
+
+
diff --git a/docs/fig/application-my-app-state-processor-api.png b/docs/fig/application-my-app-state-processor-api.png
new file mode 100644
index 0000000000000000000000000000000000000000..ca6ef6ca42ba0f0750361aa2398eb6a78f7907d8
Binary files /dev/null and b/docs/fig/application-my-app-state-processor-api.png differ
diff --git a/docs/fig/back_pressure_sampling.png b/docs/fig/back_pressure_sampling.png
index ad6ce2ffe795d2315fd382cb16203dc3dcb74de6..bd0d6ea555f9baf7f6da53b3e1d0379369415770 100644
Binary files a/docs/fig/back_pressure_sampling.png and b/docs/fig/back_pressure_sampling.png differ
diff --git a/docs/fig/database-my-app-state-processor-api.png b/docs/fig/database-my-app-state-processor-api.png
new file mode 100644
index 0000000000000000000000000000000000000000..31642fb4fab7a37bbbe5e68b34ebb8de6df2a91f
Binary files /dev/null and b/docs/fig/database-my-app-state-processor-api.png differ
diff --git a/docs/fig/detailed-mem-model.svg b/docs/fig/detailed-mem-model.svg
new file mode 100644
index 0000000000000000000000000000000000000000..215d99a7af19fff909eae3846c88baafc6ba8baa
--- /dev/null
+++ b/docs/fig/detailed-mem-model.svg
@@ -0,0 +1,21 @@
+
+
+
+
diff --git a/docs/fig/event_ingestion_processing_time.svg b/docs/fig/event_ingestion_processing_time.svg
deleted file mode 100644
index fc80d91f67a5bccafd94afab1590cab51db5cb19..0000000000000000000000000000000000000000
--- a/docs/fig/event_ingestion_processing_time.svg
+++ /dev/null
@@ -1,375 +0,0 @@
-
-
-
-
diff --git a/docs/fig/event_processing_time.svg b/docs/fig/event_processing_time.svg
new file mode 100644
index 0000000000000000000000000000000000000000..c5e8ec1a5de59cfd0aa12a134fb8bbd402abccee
--- /dev/null
+++ b/docs/fig/event_processing_time.svg
@@ -0,0 +1,18 @@
+
\ No newline at end of file
diff --git a/docs/fig/plan_visualizer.png b/docs/fig/plan_visualizer.png
index 85b8c552707d43c642089d74441fb2da4edde16c..c9b573129b218385064ddd8b78360f84e0a3d8cc 100644
Binary files a/docs/fig/plan_visualizer.png and b/docs/fig/plan_visualizer.png differ
diff --git a/docs/fig/processes.svg b/docs/fig/processes.svg
index fe83a9db7c01b0c186dd26e4d307072c67ee5d02..f2c58fc449d2bd31790641f25cb21386d9181afd 100644
--- a/docs/fig/processes.svg
+++ b/docs/fig/processes.svg
@@ -233,7 +233,7 @@ under the License.
y="460.43439"
id="text3067"
xml:space="preserve"
- style="font-size:17.55437279px;font-style:normal;font-weight:normal;text-align:start;text-anchor:start;fill:#000000;font-family:Verdana">JobManager
+ style="font-size:17.55437279px;font-style:normal;font-weight:normal;text-align:start;text-anchor:start;fill:#000000;font-family:Verdana">Flink Master
+
+
+
diff --git a/docs/fig/ssl_internal_external.svg b/docs/fig/ssl_internal_external.svg
index 04262d29cbc272872fdc39a83bff9d51b5b9cedf..42d845ecda07c10778c891ad7ef8ce6e2d8ed84e 100755
--- a/docs/fig/ssl_internal_external.svg
+++ b/docs/fig/ssl_internal_external.svg
@@ -29,7 +29,8 @@ under the License.
height="364.59875"
id="svg2"
version="1.1"
- inkscape:version="0.48.5 r10040">
+ inkscape:version="1.0beta1 (d565813, 2019-09-28)"
+ sodipodi:docname="ssl_internal_external.svg">
+ inkscape:window-width="1678"
+ inkscape:window-height="907"
+ inkscape:window-x="0"
+ inkscape:window-y="23"
+ inkscape:window-maximized="0"
+ inkscape:document-rotation="0" />
@@ -62,7 +64,7 @@ under the License.
image/svg+xml
-
+
@@ -71,266 +73,263 @@ under the License.
inkscape:groupmode="layer"
id="layer1"
transform="translate(139.76354,-243.79437)">
-
-
- Task
- Manager
-
- Task
- Manager
-
- Task
- Manager
-
- Resource
- Manager
-
- Job
- Manager
-
- Master Process
-
-
-
-
-
-
-
-
-
-
- RPC / BLOB
- Data Plane
- Data Plane
-
-
-
- REST
- Internal
- External
-
- Browser
-
- CLI
-
- Tools
-
-
- HTTP
-
- Dispatcher
-
+
+ Task
+ Manager
+
+ Task
+ Manager
+
+ Task
+ Manager
+
+ Resource
+ Manager
+
+ Job
+ Manager
+
+ Master Process
+
+
+
+
+
+
+
+
+
+
+ RPC / BLOB
+ Data Plane
+ Data Plane
+
+
+
+ REST
+ Internal
+ External
+
+ Browser
+
+ CLI
+
+ Tools
+
+
+ HTTP
+
+ Dispatcher
diff --git a/docs/fig/times_clocks.svg b/docs/fig/times_clocks.svg
deleted file mode 100644
index 2dede77501c6c802a54cc0503a3b2160438b5be1..0000000000000000000000000000000000000000
--- a/docs/fig/times_clocks.svg
+++ /dev/null
@@ -1,368 +0,0 @@
-
-
-
-
-
diff --git a/docs/flinkDev/building.md b/docs/flinkDev/building.md
index eb06d27753433b7a5f916151e2acc8d4b5fd741f..c024e6386145582068bf03f700b9a7da47b85160 100644
--- a/docs/flinkDev/building.md
+++ b/docs/flinkDev/building.md
@@ -62,13 +62,15 @@ If you want to build a PyFlink package that can be used for pip installation, yo
Then go to the root directory of flink source code and run this command to build the sdist package and wheel package:
{% highlight bash %}
-cd flink-python; python3 setup.py sdist bdist_wheel
+cd flink-python; python setup.py sdist bdist_wheel
{% endhighlight %}
+Note Python 3.5 or higher is required to build PyFlink.
+
The sdist and wheel package will be found under `./flink-python/dist/`. Either of them could be used for pip installation, such as:
{% highlight bash %}
-pip install dist/*.tar.gz
+python -m pip install dist/*.tar.gz
{% endhighlight %}
## Dependency Shading
@@ -95,49 +97,7 @@ mvn clean install
## Hadoop Versions
-Flink has optional dependencies to HDFS and YARN which are both dependencies from [Apache Hadoop](http://hadoop.apache.org). There exist many different versions of Hadoop (from both the upstream project and the different Hadoop distributions). If you are using an incompatible combination of versions, exceptions may occur.
-
-Flink can be built against any Hadoop version >= 2.4.0, but depending on the version it may be a 1 or 2 step process.
-
-### Pre-bundled versions
-
-To build against Hadoop 2.4.1, 2.6.5, 2.7.5 or 2.8.3, it is sufficient to run (e.g., for version `2.6.5`):
-
-{% highlight bash %}
-mvn clean install -DskipTests -Dhadoop.version=2.6.5
-{% endhighlight %}
-
-To package a shaded pre-packaged Hadoop jar into the distributions `/lib` directory, activate the `include-hadoop` profile:
-
-{% highlight bash %}
-mvn clean install -DskipTests -Pinclude-hadoop
-{% endhighlight %}
-
-### Custom / Vendor-specific versions
-
-If you want to build against Hadoop version that is *NOT* 2.4.1, 2.6.5, 2.7.5 or 2.8.3,
-then it is first necessary to build [flink-shaded](https://github.com/apache/flink-shaded) against this version.
-You can find the source for this project in the [Additional Components]({{ site.download_url }}#additional-components) section of the download page.
-
-Run the following command to build and install `flink-shaded` against your desired Hadoop version (e.g., for version `2.6.5-custom`):
-
-{% highlight bash %}
-mvn clean install -Dhadoop.version=2.6.5-custom
-{% endhighlight %}
-
-After this step is complete, follow the steps for [Pre-bundled versions](#pre-bundled-versions).
-
-To build Flink against a vendor specific Hadoop version, additionally activate `-Pvendor-repos` profile when building flink-shaded:
-
-{% highlight bash %}
-mvn clean install -DskipTests -Pvendor-repos -Dhadoop.version=2.6.0-cdh5.16.1
-{% endhighlight %}
-
-The `-Pvendor-repos` activates a Maven [build profile](http://maven.apache.org/guides/introduction/introduction-to-profiles.html) that includes the repositories of popular Hadoop vendors such as Cloudera, Hortonworks, or MapR.
-
-The list of supported vendor versions can be checked [here](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs?repo=cloudera).
-
-{% top %}
+Please see the [Hadoop integration section]({{ site.baseurl }}/ops/deployment/hadoop.html) on how to handle Hadoop classes and versions.
## Scala Versions
diff --git a/docs/flinkDev/building.zh.md b/docs/flinkDev/building.zh.md
index 2f000d9118dbe0dd3b77a1c722986b38c8e8def9..756d484c4d60848a2d719c42a1fb19de9f3ae21f 100644
--- a/docs/flinkDev/building.zh.md
+++ b/docs/flinkDev/building.zh.md
@@ -62,13 +62,15 @@ mvn clean install -DskipTests -Dfast
之后，进入Flink源码根目录，并执行以下命令，构建PyFlink的源码发布包和wheel包：
{% highlight bash %}
-cd flink-python; python3 setup.py sdist bdist_wheel
+cd flink-python; python setup.py sdist bdist_wheel
{% endhighlight %}
+注意事项 构建PyFlink需要Python3.5及以上的版本.
+
构建好的源码发布包和wheel包位于`./flink-python/dist/`目录下。它们均可使用pip安装,比如:
{% highlight bash %}
-pip install dist/*.tar.gz
+python -m pip install dist/*.tar.gz
{% endhighlight %}
## Dependency Shading
@@ -95,49 +97,7 @@ mvn clean install
## Hadoop Versions
-Flink has optional dependencies to HDFS and YARN which are both dependencies from [Apache Hadoop](http://hadoop.apache.org). There exist many different versions of Hadoop (from both the upstream project and the different Hadoop distributions). If you are using an incompatible combination of versions, exceptions may occur.
-
-Flink can be built against any Hadoop version >= 2.4.0, but depending on the version it may be a 1 or 2 step process.
-
-### Pre-bundled versions
-
-To build against Hadoop 2.4.1, 2.6.5, 2.7.5 or 2.8.3, it is sufficient to run (e.g., for version `2.6.5`):
-
-{% highlight bash %}
-mvn clean install -DskipTests -Dhadoop.version=2.6.5
-{% endhighlight %}
-
-To package a shaded pre-packaged Hadoop jar into the distributions `/lib` directory, activate the `include-hadoop` profile:
-
-{% highlight bash %}
-mvn clean install -DskipTests -Pinclude-hadoop
-{% endhighlight %}
-
-### Custom / Vendor-specific versions
-
-If you want to build against Hadoop version that is *NOT* 2.4.1, 2.6.5, 2.7.5 or 2.8.3,
-then it is first necessary to build [flink-shaded](https://github.com/apache/flink-shaded) against this version.
-You can find the source for this project in the [Additional Components]({{ site.download_url }}#additional-components) section of the download page.
-
-Run the following command to build and install `flink-shaded` against your desired Hadoop version (e.g., for version `2.6.5-custom`):
-
-{% highlight bash %}
-mvn clean install -Dhadoop.version=2.6.5-custom
-{% endhighlight %}
-
-After this step is complete, follow the steps for [Pre-bundled versions](#pre-bundled-versions).
-
-To build Flink against a vendor specific Hadoop version, additionally activate `-Pvendor-repos` profile when building flink-shaded:
-
-{% highlight bash %}
-mvn clean install -DskipTests -Pvendor-repos -Dhadoop.version=2.6.0-cdh5.16.1
-{% endhighlight %}
-
-The `-Pvendor-repos` activates a Maven [build profile](http://maven.apache.org/guides/introduction/introduction-to-profiles.html) that includes the repositories of popular Hadoop vendors such as Cloudera, Hortonworks, or MapR.
-
-The list of supported vendor versions can be checked [here](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs?repo=cloudera).
-
-{% top %}
+Please see the [Hadoop integration section]({{ site.baseurl }}/ops/deployment/hadoop.html) on how to handle Hadoop classes and versions.
## Scala Versions
diff --git a/docs/flinkDev/ide_setup.md b/docs/flinkDev/ide_setup.md
index bb8e67665bf9aa3cc6554dddc34fa6d30019bec9..d944bc688f086491104c3c7bdf5ff50733326f0d 100644
--- a/docs/flinkDev/ide_setup.md
+++ b/docs/flinkDev/ide_setup.md
@@ -117,6 +117,31 @@ Nevertheless please make sure that code you add/modify in these modules still co
Enable scalastyle in Intellij by selecting Settings -> Editor -> Inspections, then searching for "Scala style inspections". Also Place `"tools/maven/scalastyle-config.xml"` in the `"/.idea"` or `"/project"` directory.
+### FAQ
+
+This section lists issues that developers have run into in the past when working with IntelliJ:
+
+- Compilation fails with `invalid flag: --add-expots=java.base/sun.net.util=ALL-UNNAMED`
+
+This means that IntelliJ activated the `java11` profile despite an older JDK being used.
+Open the Maven tool window (View -> Tool Windows -> Maven), uncheck the `java11` profile and reimport the project.
+
+- Compilation fails with `cannot find symbol: symbol: method defineClass(...) location: class sun.misc.Unsafe`
+
+This means that IntelliJ is using JDK 11 for the project, but you are working on a Flink version which doesn't
+support Java 11.
+This commonly happens when you have setup IntelliJ to use JDK 11 and checkout older versions of Flink (<= 1.9).
+Open the project settings window (File -> Project Structure -> Project Settings: Project) and select JDK 8 as the project
+SDK.
+You may have to revert this after switching back to the new Flink version if you want to use JDK 11.
+
+- Examples fail with a `NoClassDefFoundError` for Flink classes.
+
+This is likely due to Flink dependencies being set to provided, resulting in them not being put automatically on the
+classpath.
+You can either tick the "Include dependencies with 'Provided' scope" box in the run configuration, or create a test
+that calls the `main()` method of the example (`provided` dependencies are available on the test classpath).
+
## Eclipse
**NOTE:** From our experience, this setup does not work with Flink
diff --git a/docs/flinkDev/ide_setup.zh.md b/docs/flinkDev/ide_setup.zh.md
index 57d76170d380ec1f460f85f8229bf4befe051038..170f26cdf945ab813b427f8ee8a536405f14ab1a 100644
--- a/docs/flinkDev/ide_setup.zh.md
+++ b/docs/flinkDev/ide_setup.zh.md
@@ -117,6 +117,31 @@ Nevertheless please make sure that code you add/modify in these modules still co
Enable scalastyle in Intellij by selecting Settings -> Editor -> Inspections, then searching for "Scala style inspections". Also Place `"tools/maven/scalastyle-config.xml"` in the `"/.idea"` or `"/project"` directory.
+### FAQ
+
+This section lists issues that developers have run into in the past when working with IntelliJ:
+
+- Compilation fails with `invalid flag: --add-expots=java.base/sun.net.util=ALL-UNNAMED`
+
+This means that IntelliJ activated the `java11` profile despite an older JDK being used.
+Open the Maven tool window (View -> Tool Windows -> Maven), uncheck the `java11` profile and reimport the project.
+
+- Compilation fails with `cannot find symbol: symbol: method defineClass(...) location: class sun.misc.Unsafe`
+
+This means that IntelliJ is using JDK 11 for the project, but you are working on a Flink version which doesn't
+support Java 11.
+This commonly happens when you have setup IntelliJ to use JDK 11 and checkout older versions of Flink (<= 1.9).
+Open the project settings window (File -> Project Structure -> Project Settings: Project) and select JDK 8 as the project
+SDK.
+You may have to revert this after switching back to the new Flink version if you want to use JDK 11.
+
+- Examples fail with a `NoClassDefFoundError` for Flink classes.
+
+This is likely due to Flink dependencies being set to provided, resulting in them not being put automatically on the
+classpath.
+You can either tick the "Include dependencies with 'Provided' scope" box in the run configuration, or create a test
+that calls the `main()` method of the example (`provided` dependencies are available on the test classpath).
+
## Eclipse
**NOTE:** From our experience, this setup does not work with Flink
diff --git a/docs/getting-started/docker-playgrounds/flink-operations-playground.md b/docs/getting-started/docker-playgrounds/flink-operations-playground.md
index 656071aeba6bcb10d53bebab201798afa8547845..1e2a569051f2bfb690651a141d574bd61bce9a8f 100644
--- a/docs/getting-started/docker-playgrounds/flink-operations-playground.md
+++ b/docs/getting-started/docker-playgrounds/flink-operations-playground.md
@@ -334,7 +334,7 @@ in its "at-least-once" mode, there is a chance that you will see some duplicate
### Upgrading & Rescaling a Job
Upgrading a Flink Job always involves two steps: First, the Flink Job is gracefully stopped with a
-[Savepoint]({{site.base_url}}/ops/state/savepoints.html). A Savepoint is a consistent snapshot of
+[Savepoint]({{ site.baseurl }}/ops/state/savepoints.html). A Savepoint is a consistent snapshot of
the complete application state at a well-defined, globally consistent point in time (similar to a
checkpoint). Second, the upgraded Flink Job is started from the Savepoint. In this context "upgrade"
can mean different things including the following:
@@ -825,4 +825,4 @@ during even-numbered minutes (e.g., during 10:12, but not during 10:13). This ca
inspecting various [network metrics]({{ site.baseurl }}/monitoring/metrics.html#default-shuffle-service)
such as `outputQueueLength` and `outPoolUsage`, and/or by using the
[backpressure monitoring]({{ site.baseurl }}/monitoring/back_pressure.html#monitoring-back-pressure)
-available in the WebUI.
\ No newline at end of file
+available in the WebUI.
diff --git a/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md b/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md
index 656071aeba6bcb10d53bebab201798afa8547845..1e2a569051f2bfb690651a141d574bd61bce9a8f 100644
--- a/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md
+++ b/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md
@@ -334,7 +334,7 @@ in its "at-least-once" mode, there is a chance that you will see some duplicate
### Upgrading & Rescaling a Job
Upgrading a Flink Job always involves two steps: First, the Flink Job is gracefully stopped with a
-[Savepoint]({{site.base_url}}/ops/state/savepoints.html). A Savepoint is a consistent snapshot of
+[Savepoint]({{ site.baseurl }}/ops/state/savepoints.html). A Savepoint is a consistent snapshot of
the complete application state at a well-defined, globally consistent point in time (similar to a
checkpoint). Second, the upgraded Flink Job is started from the Savepoint. In this context "upgrade"
can mean different things including the following:
@@ -825,4 +825,4 @@ during even-numbered minutes (e.g., during 10:12, but not during 10:13). This ca
inspecting various [network metrics]({{ site.baseurl }}/monitoring/metrics.html#default-shuffle-service)
such as `outputQueueLength` and `outPoolUsage`, and/or by using the
[backpressure monitoring]({{ site.baseurl }}/monitoring/back_pressure.html#monitoring-back-pressure)
-available in the WebUI.
\ No newline at end of file
+available in the WebUI.
diff --git a/docs/getting-started/docker-playgrounds/index.md b/docs/getting-started/docker-playgrounds/index.md
index 2f3c0291bc66fb01b07e527e75dd8cc42336f224..e3d5fd87a3a488f20baf13bc56e9332a999e6f1a 100644
--- a/docs/getting-started/docker-playgrounds/index.md
+++ b/docs/getting-started/docker-playgrounds/index.md
@@ -1,9 +1,9 @@
---
title: Docker Playgrounds
nav-id: docker-playgrounds
-nav-title: ' Docker Playgrounds'
+nav-title: 'Docker Playgrounds'
nav-parent_id: getting-started
-nav-pos: 20
+nav-pos: 10
---
-
-
-## Bundled Examples
-
-The Flink sources include many examples for Flink's different APIs:
-
-* DataStream applications ({% gh_link flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples "Java" %} / {% gh_link flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples "Scala" %})
-* DataSet applications ({% gh_link flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala "Scala" %})
-* Table API / SQL queries ({% gh_link flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala "Scala" %})
-
-These [instructions]({{ site.baseurl }}/dev/batch/examples.html#running-an-example) explain how to run the examples.
-
-## Examples on the Web
-
-There are also a few blog posts published online that discuss example applications:
-
-* [How to build stateful streaming applications with Apache Flink
-](https://www.infoworld.com/article/3293426/big-data/how-to-build-stateful-streaming-applications-with-apache-flink.html) presents an event-driven application implemented with the DataStream API and two SQL queries for streaming analytics.
-
-* [Building real-time dashboard applications with Apache Flink, Elasticsearch, and Kibana](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana) is a blog post at elastic.co showing how to build a real-time dashboard solution for streaming data analytics using Apache Flink, Elasticsearch, and Kibana.
-
-* The [Flink training website](https://training.ververica.com/) from Ververica has a number of examples. Check out the hands-on sections and the exercises.
-
-{% top %}
diff --git a/docs/getting-started/examples/index.zh.md b/docs/getting-started/examples/index.zh.md
deleted file mode 100644
index 99298a811f221260ab523f6d828c47abb26931df..0000000000000000000000000000000000000000
--- a/docs/getting-started/examples/index.zh.md
+++ /dev/null
@@ -1,49 +0,0 @@
----
-title: 示例
-nav-id: examples
-nav-title: ' 示例'
-nav-parent_id: getting-started
-nav-pos: 40
-nav-show_overview: true
----
-
-
-
-## 附带示例
-
-在Flink源文件中，包含了许多 Flink 不同 API 接口的代码示例：
-
-* DataStream 应用 ({% gh_link flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples "Java" %} / {% gh_link flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples "Scala" %})
-* DataSet 应用 ({% gh_link flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala "Scala" %})
-* Table API / SQL 查询 ({% gh_link flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala "Scala" %})
-
-这些[代码示例]({{ site.baseurl }}/zh/dev/batch/examples.html#running-an-example)清晰的解释了如何运行一个 Flink 程序。
-
-## 网上示例
-
-在网上也有一些博客讨论了 Flink 应用示例
-
-* [如何使用 Apache Flink 构建有状态流数据应用](https://www.infoworld.com/article/3293426/big-data/how-to-build-stateful-streaming-applications-with-apache-flink.html)，这篇博客提供了基于 DataStream API 以及两个用于流数据分析的 SQL 查询实现的事件驱动的应用程序。
-
-* [使用 Apache Flink、Elasticsearch 和 Kibana 构建实时仪表板应用程序](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana)是 Elastic 的一篇博客，它向我们提供了一种基于 Apache Flink、Elasticsearch 和 Kibana 构建流数据分析实时仪表板的解决方案。
-
-* 来自 Ververica 的 [Flink 学习网站](https://training.ververica.com/)也有许多示例。你可以从中选取能够亲自实践的部分，并加以练习。
-
-{% top %}
diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md
index 6f02e884f6d67c88e81b874dca9a598cf19d59b7..7d81c48c0ad8656348d744dd0e3afba6ebb9b57a 100644
--- a/docs/getting-started/index.md
+++ b/docs/getting-started/index.md
@@ -6,6 +6,7 @@ nav-parent_id: root
section-break: true
nav-show_overview: true
nav-pos: 1
+always-expand: true
---
-There are many ways to get started with Apache Flink. Which one is the best for you depends on your goal and prior experience.
+There are many ways to get started with Apache Flink. Which one is the best for
+you depends on your goal and prior experience:
+
+* take a look at the **Docker Playgrounds** for a docker-based introduction to
+ specific Flink concepts
+* explore on of the **Code Walkthroughs** if you want to get an end-to-end
+ introduction to using one of the Flink APIs
+* use **Project Setup** if you already know the basics of Flink but want to get a
+ project setup template for Java or Scala and need help setting up
+ dependencies
### Taking a first look at Flink
diff --git a/docs/getting-started/index.zh.md b/docs/getting-started/index.zh.md
index 8d81d243587026b228c324f722a2e818cdbe53b7..cb81bbc8bab041ef0e9c14c999bcde36e8710283 100644
--- a/docs/getting-started/index.zh.md
+++ b/docs/getting-started/index.zh.md
@@ -6,6 +6,7 @@ nav-parent_id: root
section-break: true
nav-show_overview: true
nav-pos: 1
+always-expand: true
---
-* [**DataStream API 示例**](./tutorials/datastream_api.html) 展示了如何编写一个基本的 DataStream 应用程序。 DataStream API 是 Flink 的主要抽象，用于通过 Java 或 Scala 实现具有复杂时间语义的有状态数据流处理的应用程序。
+* [**DataStream API 示例**](./walkthroughs/datastream_api.html) 展示了如何编写一个基本的 DataStream 应用程序。 DataStream API 是 Flink 的主要抽象，用于通过 Java 或 Scala 实现具有复杂时间语义的有状态数据流处理的应用程序。
* [**Table API 示例**](./walkthroughs/table_api.html) 演示了如何在批处中使用简单的 Table API 进行查询，以及如何将其扩展为流处理中的查询。Table API 是 Flink 的语言嵌入式关系 API，用于在 Java 或 Scala 中编写类 SQL 的查询，这些查询会自动进行优化。Table API 查询可以使用一致的语法和语义同时在批处理或流数据上运行。
diff --git a/docs/dev/projectsetup/dependencies.md b/docs/getting-started/project-setup/dependencies.md
similarity index 94%
rename from docs/dev/projectsetup/dependencies.md
rename to docs/getting-started/project-setup/dependencies.md
index ce71f51d52beaab42faecf2d96caf2381f69500a..51f1dd020d22d372179559aa9df5aceeb7c835d0 100644
--- a/docs/dev/projectsetup/dependencies.md
+++ b/docs/getting-started/project-setup/dependencies.md
@@ -1,6 +1,6 @@
---
title: "Configuring Dependencies, Connectors, Libraries"
-nav-parent_id: projectsetup
+nav-parent_id: project-setup
nav-pos: 2
---
- {% highlight bash %}
- bash -c "$(curl https://flink.apache.org/q/gradle-quickstart.sh)" -- {{site.version}} {{site.scala_version}}
- {% endhighlight %}
+{% highlight bash %}
+bash -c "$(curl https://flink.apache.org/q/gradle-quickstart.sh)" -- {{site.version}} {{site.scala_version}}
+{% endhighlight %}
This allows you to name your newly created project. It will interactively ask
you for the project name, organization (also used for the package name), project version,
Scala and Flink version.
@@ -301,7 +322,7 @@ quickstart/
│ ├── BatchJob.java
│ └── StreamingJob.java
└── resources
- └── log4j.properties
+ └── log4j2.properties
{% endhighlight %}
The sample project is a __Gradle project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program.
@@ -345,7 +366,7 @@ For a complete overview over the APIs, have a look at the
[DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and
[DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections.
-[Here]({{ site.baseurl }}/getting-started/tutorials/local_setup.html) you can find out how to run an application outside the IDE on a local cluster.
+[Here]({{ site.baseurl }}/ops/deployment/local.html) you can find out how to run an application outside the IDE on a local cluster.
If you have any trouble, ask on our
[Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/).
diff --git a/docs/dev/projectsetup/java_api_quickstart.zh.md b/docs/getting-started/project-setup/java_api_quickstart.zh.md
similarity index 85%
rename from docs/dev/projectsetup/java_api_quickstart.zh.md
rename to docs/getting-started/project-setup/java_api_quickstart.zh.md
index 4a89491fa6824ad24a0138cd2d3701e798854236..a5a0493a983781ac78f77527e23eaf764b38ef4c 100644
--- a/docs/dev/projectsetup/java_api_quickstart.zh.md
+++ b/docs/getting-started/project-setup/java_api_quickstart.zh.md
@@ -1,7 +1,7 @@
---
title: "Java 项目模板"
nav-title: Java 项目模板
-nav-parent_id: projectsetup
+nav-parent_id: project-setup
nav-pos: 0
---
@@ -293,7 +314,7 @@ quickstart/
│ ├── BatchJob.java
│ └── StreamingJob.java
└── resources
- └── log4j.properties
+ └── log4j2.properties
{% endhighlight %}
示例项目是一个 __Gradle 项目__，它包含了两个类：_StreamingJob_ 和 _BatchJob_ 是 *DataStream* 和 *DataSet* 程序的基础骨架程序。
@@ -332,7 +353,7 @@ __注意：__ 如果你使用其他类而不是 *StreamingJob* 作为应用程
[DataStream API]({{ site.baseurl }}/zh/dev/datastream_api.html) 和
[DataSet API]({{ site.baseurl }}/zh/dev/batch/index.html) 章节。
-在[这里]({{ site.baseurl }}/zh/getting-started/tutorials/local_setup.html)，你可以找到如何在 IDE 之外的本地集群中运行应用程序。
+在[这里]({{ site.baseurl }}/zh/ops/deployment/local.html)，你可以找到如何在 IDE 之外的本地集群中运行应用程序。
如果你有任何问题，请发信至我们的[邮箱列表](http://mail-archives.apache.org/mod_mbox/flink-user/)，我们很乐意提供帮助。
diff --git a/docs/dev/projectsetup/scala_api_quickstart.md b/docs/getting-started/project-setup/scala_api_quickstart.md
similarity index 84%
rename from docs/dev/projectsetup/scala_api_quickstart.md
rename to docs/getting-started/project-setup/scala_api_quickstart.md
index b03518ace218c11519001770b5d03b742c6861cb..82e13683433cb9c334b5038d4a178a5b0ecf616f 100644
--- a/docs/dev/projectsetup/scala_api_quickstart.md
+++ b/docs/getting-started/project-setup/scala_api_quickstart.md
@@ -1,7 +1,7 @@
---
title: "Project Template for Scala"
nav-title: Project Template for Scala
-nav-parent_id: projectsetup
+nav-parent_id: project-setup
nav-pos: 1
---
-
-If you want to run Flink locally on a Windows machine you need to [download](https://flink.apache.org/downloads.html) and unpack the binary Flink distribution. After that you can either use the **Windows Batch** file (`.bat`), or use **Cygwin** to run the Flink Jobmanager.
-
-## Starting with Windows Batch Files
-
-To start Flink in from the *Windows Command Line*, open the command window, navigate to the `bin/` directory of Flink and run `start-cluster.bat`.
-
-Note: The ``bin`` folder of your Java Runtime Environment must be included in Window's ``%PATH%`` variable. Follow this [guide](http://www.java.com/en/download/help/path.xml) to add Java to the ``%PATH%`` variable.
-
-{% highlight bash %}
-$ cd flink
-$ cd bin
-$ start-cluster.bat
-Starting a local cluster with one JobManager process and one TaskManager process.
-You can terminate the processes via CTRL-C in the spawned shell windows.
-Web interface by default on http://localhost:8081/.
-{% endhighlight %}
-
-After that, you need to open a second terminal to run jobs using `flink.bat`.
-
-{% top %}
-
-## Starting with Cygwin and Unix Scripts
-
-With *Cygwin* you need to start the Cygwin Terminal, navigate to your Flink directory and run the `start-cluster.sh` script:
-
-{% highlight bash %}
-$ cd flink
-$ bin/start-cluster.sh
-Starting cluster.
-{% endhighlight %}
-
-{% top %}
-
-## Installing Flink from Git
-
-If you are installing Flink from the git repository and you are using the Windows git shell, Cygwin can produce a failure similar to this one:
-
-{% highlight bash %}
-c:/flink/bin/start-cluster.sh: line 30: $'\r': command not found
-{% endhighlight %}
-
-This error occurs because git is automatically transforming UNIX line endings to Windows style line endings when running in Windows. The problem is that Cygwin can only deal with UNIX style line endings. The solution is to adjust the Cygwin settings to deal with the correct line endings by following these three steps:
-
-1. Start a Cygwin shell.
-
-2. Determine your home directory by entering
-
- {% highlight bash %}
- cd; pwd
- {% endhighlight %}
-
- This will return a path under the Cygwin root path.
-
-3. Using NotePad, WordPad or a different text editor open the file `.bash_profile` in the home directory and append the following: (If the file does not exist you will have to create it)
-
-{% highlight bash %}
-export SHELLOPTS
-set -o igncr
-{% endhighlight %}
-
-Save the file and open a new bash shell.
-
-{% top %}
diff --git a/docs/getting-started/tutorials/flink_on_windows.zh.md b/docs/getting-started/tutorials/flink_on_windows.zh.md
deleted file mode 100644
index 513eac0e3eecbea88f6a24cedac8b1a50a69afe8..0000000000000000000000000000000000000000
--- a/docs/getting-started/tutorials/flink_on_windows.zh.md
+++ /dev/null
@@ -1,87 +0,0 @@
----
-title: "在 Windows 上运行 Flink"
-nav-parent_id: setuptutorials
-nav-pos: 30
----
-
-
-If you want to run Flink locally on a Windows machine you need to [download](https://flink.apache.org/downloads.html) and unpack the binary Flink distribution. After that you can either use the **Windows Batch** file (`.bat`), or use **Cygwin** to run the Flink Jobmanager.
-
-## Starting with Windows Batch Files
-
-To start Flink in from the *Windows Command Line*, open the command window, navigate to the `bin/` directory of Flink and run `start-cluster.bat`.
-
-Note: The ``bin`` folder of your Java Runtime Environment must be included in Window's ``%PATH%`` variable. Follow this [guide](http://www.java.com/en/download/help/path.xml) to add Java to the ``%PATH%`` variable.
-
-{% highlight bash %}
-$ cd flink
-$ cd bin
-$ start-cluster.bat
-Starting a local cluster with one JobManager process and one TaskManager process.
-You can terminate the processes via CTRL-C in the spawned shell windows.
-Web interface by default on http://localhost:8081/.
-{% endhighlight %}
-
-After that, you need to open a second terminal to run jobs using `flink.bat`.
-
-{% top %}
-
-## Starting with Cygwin and Unix Scripts
-
-With *Cygwin* you need to start the Cygwin Terminal, navigate to your Flink directory and run the `start-cluster.sh` script:
-
-{% highlight bash %}
-$ cd flink
-$ bin/start-cluster.sh
-Starting cluster.
-{% endhighlight %}
-
-{% top %}
-
-## Installing Flink from Git
-
-If you are installing Flink from the git repository and you are using the Windows git shell, Cygwin can produce a failure similar to this one:
-
-{% highlight bash %}
-c:/flink/bin/start-cluster.sh: line 30: $'\r': command not found
-{% endhighlight %}
-
-This error occurs because git is automatically transforming UNIX line endings to Windows style line endings when running in Windows. The problem is that Cygwin can only deal with UNIX style line endings. The solution is to adjust the Cygwin settings to deal with the correct line endings by following these three steps:
-
-1. Start a Cygwin shell.
-
-2. Determine your home directory by entering
-
- {% highlight bash %}
- cd; pwd
- {% endhighlight %}
-
- This will return a path under the Cygwin root path.
-
-3. Using NotePad, WordPad or a different text editor open the file `.bash_profile` in the home directory and append the following: (If the file does not exist you will have to create it)
-
-{% highlight bash %}
-export SHELLOPTS
-set -o igncr
-{% endhighlight %}
-
-Save the file and open a new bash shell.
-
-{% top %}
diff --git a/docs/getting-started/walkthroughs/datastream_api.md b/docs/getting-started/walkthroughs/datastream_api.md
index 8676ae47d1005c8a5dadbd97de995d34406414db..7aef86e804ea570549e133a3cb46a2fb1f70d946 100644
--- a/docs/getting-started/walkthroughs/datastream_api.md
+++ b/docs/getting-started/walkthroughs/datastream_api.md
@@ -54,7 +54,7 @@ In particular, Apache Flink's [user mailing list](https://flink.apache.org/commu
If you want to follow along, you will require a computer with:
-* Java 8
+* Java 8 or 11
* Maven
A provided Flink Maven Archetype will create a skeleton project with all the necessary dependencies quickly, so you only need to focus on filling out the business logic.
@@ -95,7 +95,26 @@ $ mvn archetype:generate \
{% unless site.is_stable %}
- Note: For Maven 3.0 or higher, it is no longer possible to specify the repository (-DarchetypeCatalog) via the commandline. If you wish to use the snapshot repository, you need to add a repository entry to your settings.xml. For details about this change, please refer to Maven official document
+ Note: For Maven 3.0 or higher, it is no longer possible to specify the repository (-DarchetypeCatalog) via the command line. For details about this change, please refer to Maven official document
+ If you wish to use the snapshot repository, you need to add a repository entry to your settings.xml. For example:
+{% highlight bash %}
+
+
+ apache
+
+
+
+ apache
+
+
+ apache-snapshots
+ https://repository.apache.org/content/repositories/snapshots/
+
+
+
+
+
+{% endhighlight %}
{% endunless %}
@@ -501,65 +520,65 @@ Below, you can see an example of how you can use a flag state to track potential
{% highlight java %}
- @Override
- public void processElement(
- Transaction transaction,
- Context context,
- Collector collector) throws Exception {
-
- // Get the current state for the current key
- Boolean lastTransactionWasSmall = flagState.value();
+@Override
+public void processElement(
+ Transaction transaction,
+ Context context,
+ Collector collector) throws Exception {
- // Check if the flag is set
- if (lastTransactionWasSmall != null) {
- if (transaction.getAmount() > LARGE_AMOUNT) {
- // Output an alert downstream
- Alert alert = new Alert();
- alert.setId(transaction.getAccountId());
+ // Get the current state for the current key
+ Boolean lastTransactionWasSmall = flagState.value();
- collector.collect(alert);
- }
+ // Check if the flag is set
+ if (lastTransactionWasSmall != null) {
+ if (transaction.getAmount() > LARGE_AMOUNT) {
+ // Output an alert downstream
+ Alert alert = new Alert();
+ alert.setId(transaction.getAccountId());
- // Clean up our state
- flagState.clear();
+ collector.collect(alert);
}
- if (transaction.getAmount() < SMALL_AMOUNT) {
- // Set the flag to true
- flagState.update(true);
- }
+ // Clean up our state
+ flagState.clear();
+ }
+
+ if (transaction.getAmount() < SMALL_AMOUNT) {
+ // Set the flag to true
+ flagState.update(true);
}
+}
{% endhighlight %}
{% highlight scala %}
- override def processElement(
- transaction: Transaction,
- context: KeyedProcessFunction[Long, Transaction, Alert]#Context,
- collector: Collector[Alert]): Unit = {
-
- // Get the current state for the current key
- val lastTransactionWasSmall = flagState.value
-
- // Check if the flag is set
- if (lastTransactionWasSmall != null) {
- if (transaction.getAmount > FraudDetector.LARGE_AMOUNT) {
- // Output an alert downstream
- val alert = new Alert
- alert.setId(transaction.getAccountId)
-
- collector.collect(alert)
- }
- // Clean up our state
- flagState.clear()
+override def processElement(
+ transaction: Transaction,
+ context: KeyedProcessFunction[Long, Transaction, Alert]#Context,
+ collector: Collector[Alert]): Unit = {
+
+ // Get the current state for the current key
+ val lastTransactionWasSmall = flagState.value
+
+ // Check if the flag is set
+ if (lastTransactionWasSmall != null) {
+ if (transaction.getAmount > FraudDetector.LARGE_AMOUNT) {
+ // Output an alert downstream
+ val alert = new Alert
+ alert.setId(transaction.getAccountId)
+
+ collector.collect(alert)
}
+ // Clean up our state
+ flagState.clear()
+ }
- if (transaction.getAmount < FraudDetector.SMALL_AMOUNT) {
- // set the flag to true
- flagState.update(true)
- }
+ if (transaction.getAmount < FraudDetector.SMALL_AMOUNT) {
+ // set the flag to true
+ flagState.update(true)
}
+}
{% endhighlight %}
@@ -593,21 +612,21 @@ To cancel a timer, you have to remember what time it is set for, and remembering
diff --git a/docs/getting-started/walkthroughs/datastream_api.zh.md b/docs/getting-started/walkthroughs/datastream_api.zh.md
index 8676ae47d1005c8a5dadbd97de995d34406414db..f21004d495a01a3e3ffe04bfc8c7cf2a3c5872a1 100644
--- a/docs/getting-started/walkthroughs/datastream_api.zh.md
+++ b/docs/getting-started/walkthroughs/datastream_api.zh.md
@@ -24,43 +24,43 @@ specific language governing permissions and limitations
under the License.
-->
-Apache Flink offers a DataStream API for building robust, stateful streaming applications.
-It provides fine-grained control over state and time, which allows for the implementation of advanced event-driven systems.
-In this step-by-step guide you'll learn how to build a stateful streaming application with Flink's DataStream API.
+Apache Flink 提供了 DataStream API 来实现稳定可靠的、有状态的流处理应用程序。
+Flink 支持对状态和时间的细粒度控制，以此来实现复杂的事件驱动数据处理系统。
+这个入门指导手册讲述了如何通过 Flink DataStream API 来实现一个有状态流处理程序。
* This will be replaced by the TOC
{:toc}
-## What Are You Building?
+## 你要搭建一个什么系统
-Credit card fraud is a growing concern in the digital age.
-Criminals steal credit card numbers by running scams or hacking into insecure systems.
-Stolen numbers are tested by making one or more small purchases, often for a dollar or less.
-If that works, they then make more significant purchases to get items they can sell or keep for themselves.
+在当今数字时代，信用卡欺诈行为越来越被重视。
+罪犯可以通过诈骗或者入侵安全级别较低系统来盗窃信用卡卡号。
+用盗得的信用卡进行很小额度的例如一美元或者更小额度的消费进行测试。
+如果测试消费成功，那么他们就会用这个信用卡进行大笔消费，来购买一些他们希望得到的，或者可以倒卖的财物。
-In this tutorial, you will build a fraud detection system for alerting on suspicious credit card transactions.
-Using a simple set of rules, you will see how Flink allows us to implement advanced business logic and act in real-time.
+在这个教程中，你将会建立一个针对可疑信用卡交易行为的反欺诈检测系统。
+通过使用一组简单的规则，你将了解到 Flink 如何为我们实现复杂业务逻辑并实时执行。
-## Prerequisites
+## 准备条件
-This walkthrough assumes that you have some familiarity with Java or Scala, but you should be able to follow along even if you are coming from a different programming language.
+这个代码练习假定你对 Java 或 Scala 有一定的了解，当然，如果你之前使用的是其他开发语言，你也应该能够跟随本教程进行学习。
-## Help, I’m Stuck!
+## 困难求助
-If you get stuck, check out the [community support resources](https://flink.apache.org/gettinghelp.html).
-In particular, Apache Flink's [user mailing list](https://flink.apache.org/community.html#mailing-lists) is consistently ranked as one of the most active of any Apache project and a great way to get help quickly.
+如果遇到困难，可以参考 [社区支持资源](https://flink.apache.org/zh/gettinghelp.html)。
+当然也可以在邮件列表提问，Flink 的 [用户邮件列表](https://flink.apache.org/zh/community.html#mailing-lists) 一直被评为所有Apache项目中最活跃的一个，这也是快速获得帮助的好方法。
-## How to Follow Along
+## 怎样跟着教程练习
-If you want to follow along, you will require a computer with:
+首先，你需要在你的电脑上准备以下环境：
-* Java 8
-* Maven
+* Java 8 or 11
+* Maven
-A provided Flink Maven Archetype will create a skeleton project with all the necessary dependencies quickly, so you only need to focus on filling out the business logic.
-These dependencies include `flink-streaming-java` which is the core dependency for all Flink streaming applications and `flink-walkthrough-common` that has data generators and other classes specific to this walkthrough.
+一个准备好的 Flink Maven Archetype 能够快速创建一个包含了必要依赖的 Flink 程序骨架，基于此，你可以把精力集中在编写业务逻辑上即可。
+这些已包含的依赖包括 `flink-streaming-java`、`flink-walkthrough-common` 等，他们分别是 Flink 应用程序的核心依赖项和这个代码练习需要的数据生成器，当然还包括其他本代码练习所依赖的类。
-{% panel **Note:** Each code block within this walkthrough may not contain the full surrounding class for brevity. The full code is available [at the bottom of the page](#final-application). %}
+{% panel **说明:** 为简洁起见，本练习中的代码块中可能不包含完整的类路径。完整的类路径可以在文档底部 [链接](#完整的程序) 中找到。 %}
- Note: For Maven 3.0 or higher, it is no longer possible to specify the repository (-DarchetypeCatalog) via the commandline. If you wish to use the snapshot repository, you need to add a repository entry to your settings.xml. For details about this change, please refer to Maven official document
+ 注意：Maven 3.0 及更高版本，不再支持通过命令行指定仓库（-DarchetypeCatalog）。有关这个改动的详细信息，
+ 请参阅 Maven 官方文档
+ 如果你希望使用快照仓库，则需要在 settings.xml 文件中添加一个仓库条目。例如：
+{% highlight bash %}
+
+
+ apache
+
+
+
+ apache
+
+
+ apache-snapshots
+ https://repository.apache.org/content/repositories/snapshots/
+
+
+
+
+
+{% endhighlight %}
{% endunless %}
-You can edit the `groupId`, `artifactId` and `package` if you like. With the above parameters,
-Maven will create a folder named `frauddetection` that contains a project with all the dependencies to complete this tutorial.
-After importing the project into your editor, you can find a file `FraudDetectionJob.java` (or `FraudDetectionJob.scala`) with the following code which you can run directly inside your IDE.
-Try setting break points through out the data stream and run the code in DEBUG mode to get a feeling for how everything works.
+你可以根据自己的情况修改 `groupId`、 `artifactId` 和 `package`。通过这三个参数，
+Maven 将会创建一个名为 `frauddetection` 的文件夹，包含了所有依赖的整个工程项目将会位于该文件夹下。
+将工程目录导入到你的开发环境之后，你可以找到 `FraudDetectionJob.java` （或 `FraudDetectionJob.scala`） 代码文件，文件中的代码如下所示。你可以在 IDE 中直接运行这个文件。
+同时，你可以试着在数据流中设置一些断点或者以 DEBUG 模式来运行程序，体验 Flink 是如何运行的。
-## Breaking Down the Code
+## 代码分析
-Let's walk step-by-step through the code of these two files. The `FraudDetectionJob` class defines the data flow of the application and the `FraudDetector` class defines the business logic of the function that detects fraudulent transactions.
+让我们一步步地来分析一下这两个代码文件。`FraudDetectionJob` 类定义了程序的数据流，而 `FraudDetector` 类定义了欺诈交易检测的业务逻辑。
-We start describing how the Job is assembled in the `main` method of the `FraudDetectionJob` class.
+下面我们开始讲解整个 Job 是如何组装到 `FraudDetectionJob` 类的 `main` 函数中的。
-#### The Execution Environment
+#### 执行环境
-The first line sets up your `StreamExecutionEnvironment`.
-The execution environment is how you set properties for your Job, create your sources, and finally trigger the execution of the Job.
+第一行的 `StreamExecutionEnvironment` 用于设置你的执行环境。
+任务执行环境用于定义任务的属性、创建数据源以及最终启动任务的执行。
@@ -270,12 +290,12 @@ val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnv
-#### Creating a Source
+#### 创建数据源
-Sources ingest data from external systems, such as Apache Kafka, Rabbit MQ, or Apache Pulsar, into Flink Jobs.
-This walkthrough uses a source that generates an infinite stream of credit card transactions for you to process.
-Each transaction contains an account ID (`accountId`), timestamp (`timestamp`) of when the transaction occurred, and US$ amount (`amount`).
-The `name` attached to the source is just for debugging purposes, so if something goes wrong, we will know where the error originated.
+数据源从外部系统例如 Apache Kafka、Rabbit MQ 或者 Apache Pulsar 接收数据，然后将数据送到 Flink 程序中。
+这个代码练习使用的是一个能够无限循环生成信用卡模拟交易数据的数据源。
+每条交易数据包括了信用卡 ID （`accountId`），交易发生的时间 （`timestamp`） 以及交易的金额（`amount`）。
+绑定到数据源上的 `name` 属性是为了调试方便，如果发生一些异常，我们能够通过它快速定位问题发生在哪里。
@@ -296,13 +316,13 @@ val transactions: DataStream[Transaction] = env
-#### Partitioning Events & Detecting Fraud
+#### 对事件分区 & 欺诈检测
-The `transactions` stream contains a lot of transactions from a large number of users, such that it needs to be processed in parallel my multiple fraud detection tasks. Since fraud occurs on a per-account basis, you must ensure that all transactions for the same account are processed by the same parallel task of the fraud detector operator.
+`transactions` 这个数据流包含了大量的用户交易数据，需要被划分到多个并发上进行欺诈检测处理。由于欺诈行为的发生是基于某一个账户的，所以，必须要要保证同一个账户的所有交易行为数据要被同一个并发的 task 进行处理。
-To ensure that the same physical task processes all records for a particular key, you can partition a stream using `DataStream#keyBy`.
-The `process()` call adds an operator that applies a function to each partitioned element in the stream.
-It is common to say the operator immediately after a `keyBy`, in this case `FraudDetector`, is executed within a _keyed context_.
+为了保证同一个 task 处理同一个 key 的所有数据，你可以使用 `DataStream#keyBy` 对流进行分区。
+`process()` 函数对流绑定了一个操作，这个操作将会对流上的每一个消息调用所定义好的函数。
+通常，一个操作会紧跟着 `keyBy` 被调用，在这个例子中，这个操作是`FraudDetector`，该操作是在一个 _keyed context_ 上执行的。
@@ -325,10 +345,10 @@ val alerts: DataStream[Alert] = transactions
-#### Outputting Results
-
-A sink writes a `DataStream` to an external system; such as Apache Kafka, Cassandra, and AWS Kinesis.
-The `AlertSink` logs each `Alert` record with log level **INFO**, instead of writing it to persistent storage, so you can easily see your results.
+#### 输出结果
+
+sink 会将 `DataStream` 写出到外部系统，例如 Apache Kafka、Cassandra 或者 AWS Kinesis 等。
+`AlertSink` 使用 **INFO** 的日志级别打印每一个 `Alert` 的数据记录，而不是将其写入持久存储，以便你可以方便地查看结果。
-#### Executing the Job
+#### 运行作业
-Flink applications are built lazily and shipped to the cluster for execution only once fully formed.
-Call `StreamExecutionEnvironment#execute` to begin the execution of our Job and give it a name.
+Flink 程序是懒加载的，并且只有在完全搭建好之后，才能够发布到集群上执行。
+调用 `StreamExecutionEnvironment#execute` 时给任务传递一个任务名参数，就可以开始运行任务。
-#### The Fraud Detector
+#### 欺诈检测器
-The fraud detector is implemented as a `KeyedProcessFunction`.
-Its method `KeyedProcessFunction#processElement` is called for every transaction event.
-This first version produces an alert on every transaction, which some may say is overly conservative.
+欺诈检查类 `FraudDetector` 是 `KeyedProcessFunction` 接口的一个实现。
+他的方法 `KeyedProcessFunction#processElement` 将会在每个交易事件上被调用。
+这个程序里边会对每笔交易发出警报，有人可能会说这做报过于保守了。
-The next steps of this tutorial will guide you to expand the fraud detector with more meaningful business logic.
+本教程的后续步骤将指导你对这个欺诈检测器进行更有意义的业务逻辑扩展。
@@ -385,7 +405,7 @@ public class FraudDetector extends KeyedProcessFunction collector) throws Exception {
-
+
Alert alert = new Alert();
alert.setId(transaction.getAccountId());
@@ -422,38 +442,39 @@ class FraudDetector extends KeyedProcessFunction[Long, Transaction, Alert] {
-## Writing a Real Application (v1)
+## 实现一个真正的应用程序
-For the first version, the fraud detector should output an alert for any account that makes a small transaction immediately followed by a large one. Where small is anything less than $1.00 and large is more than $500.
-Imagine your fraud detector processes the following stream of transactions for a particular account.
+我们先实现第一版报警程序，对于一个账户，如果出现小于 $1 美元的交易后紧跟着一个大于 $500 的交易，就输出一个报警信息。
+
+假设你的欺诈检测器所处理的交易数据如下：
-Transactions 3 and 4 should be marked as fraudulent because it is a small transaction, $0.09, followed by a large one, $510.
-Alternatively, transactions 7, 8, and 9 are not fraud because the small amount of $0.02 is not immediately followed by the large one; instead, there is an intermediate transaction that breaks the pattern.
+交易 3 和交易 4 应该被标记为欺诈行为，因为交易 3 是一个 $0.09 的小额交易，而紧随着的交易 4 是一个 $510 的大额交易。
+另外，交易 7、8 和 交易 9 就不属于欺诈交易了，因为在交易 7 这个 $0.02 的小额交易之后，并没有跟随一个大额交易，而是一个金额适中的交易，这使得交易 7 到 交易 9 不属于欺诈行为。
-To do this, the fraud detector must _remember_ information across events; a large transaction is only fraudulent if the previous one was small.
-Remembering information across events requires [state]({{ site.baseurl }}/concepts/glossary.html#managed-state), and that is why we decided to use a [KeyedProcessFunction]({{ site.baseurl }}/dev/stream/operators/process_function.html).
-It provides fine-grained control over both state and time, which will allow us to evolve our algorithm with more complex requirements throughout this walkthrough.
+欺诈检测器需要在多个交易事件之间记住一些信息。仅当一个大额的交易紧随一个小额交易的情况发生时，这个大额交易才被认为是欺诈交易。
+在多个事件之间存储信息就需要使用到 [状态]({{ site.baseurl }}/zh/concepts/glossary.html#managed-state)，这也是我们选择使用 [KeyedProcessFunction]({{ site.baseurl }}/zh/dev/stream/operators/process_function.html) 的原因。
+它能够同时提供对状态和时间的细粒度操作，这使得我们能够在接下来的代码练习中实现更复杂的算法。
-The most straightforward implementation is a boolean flag that is set whenever a small transaction is processed.
-When a large transaction comes through, you can simply check if the flag is set for that account.
+最直接的实现方式是使用一个 boolean 型的标记状态来表示是否刚处理过一个小额交易。
+当处理到该账户的一个大额交易时，你只需要检查这个标记状态来确认上一个交易是是否小额交易即可。
-However, merely implementing the flag as a member variable in the `FraudDetector` class will not work.
-Flink processes the transactions of multiple accounts with the same object instance of `FraudDetector`, which means if accounts A and B are routed through the same instance of `FraudDetector`, a transaction for account A could set the flag to true and then a transaction for account B could set off a false alert.
-We could of course use a data structure like a `Map` to keep track of the flags for individual keys, however, a simple member variable would not be fault-tolerant and all its information be lost in case of a failure.
-Hence, the fraud detector would possibly miss alerts if the application ever had to restart to recover from a failure.
+然而，仅使用一个标记作为 `FraudDetector` 的类成员来记录账户的上一个交易状态是不准确的。
+Flink 会在同一个 `FraudDetector` 的并发实例中处理多个账户的交易数据，假设，当账户 A 和账户 B 的数据被分发的同一个并发实例上处理时，账户 A 的小额交易行为可能会将标记状态设置为真，随后账户 B 的大额交易可能会被误判为欺诈交易。
+当然，我们可以使用如 `Map` 这样的数据结构来保存每一个账户的状态，但是常规的类成员变量是无法做到容错处理的，当任务失败重启后，之前的状态信息将会丢失。
+这样的话，如果程序曾出现过失败重启的情况，将会漏掉一些欺诈报警。
-To address these challenges, Flink provides primitives for fault-tolerant state that are almost as easy to use as regular member variables.
+为了应对这个问题，Flink 提供了一套支持容错状态的原语，这些原语几乎与常规成员变量一样易于使用。
-The most basic type of state in Flink is [ValueState]({{ site.baseurl }}/dev/stream/state/state.html#using-managed-keyed-state), a data type that adds fault tolerance to any variable it wraps.
-`ValueState` is a form of _keyed state_, meaning it is only available in operators that are applied in a _keyed context_; any operator immediately following `DataStream#keyBy`.
-A _keyed state_ of an operator is automatically scoped to the key of the record that is currently processed.
-In this example, the key is the account id for the current transaction (as declared by `keyBy()`), and `FraudDetector` maintains an independent state for each account.
-`ValueState` is created using a `ValueStateDescriptor` which contains metadata about how Flink should manage the variable. The state should be registered before the function starts processing data.
-The right hook for this is the `open()` method.
+Flink 中最基础的状态类型是 [ValueState]({{ site.baseurl }}/zh/dev/stream/state/state.html#using-managed-keyed-state)，这是一种能够为被其封装的变量添加容错能力的类型。
+`ValueState` 是一种 _keyed state_，也就是说它只能被用于 _keyed context_ 提供的 operator 中，即所有能够紧随 `DataStream#keyBy` 之后被调用的operator。
+一个 operator 中的 _keyed state_ 的作用域默认是属于它所属的 key 的。
+这个例子中，key 就是当前正在处理的交易行为所属的信用卡账户（key 传入 keyBy() 函数调用），而 `FraudDetector` 维护了每个帐户的标记状态。
+`ValueState` 需要使用 `ValueStateDescriptor` 来创建，`ValueStateDescriptor` 包含了 Flink 如何管理变量的一些元数据信息。状态在使用之前需要先被注册。
+状态需要使用 `open()` 函数来注册状态。
-`ValueState` is a wrapper class, similar to `AtomicReference` or `AtomicLong` in the Java standard library.
-It provides three methods for interacting with its contents; `update` sets the state, `value` gets the current value, and `clear` deletes its contents.
-If the state for a particular key is empty, such as at the beginning of an application or after calling `ValueState#clear`, then `ValueState#value` will return `null`.
-Modifications to the object returned by `ValueState#value` are not guaranteed to be recognized by the system, and so all changes must be performed with `ValueState#update`.
-Otherwise, fault tolerance is managed automatically by Flink under the hood, and so you can interact with it like with any standard variable.
+`ValueState` 是一个包装类，类似于 Java 标准库里边的 `AtomicReference` 和 `AtomicLong`。
+它提供了三个用于交互的方法。`update` 用于更新状态，`value` 用于获取状态值，还有 `clear` 用于清空状态。
+如果一个 key 还没有状态，例如当程序刚启动或者调用过 `ValueState#clear` 方法时，`ValueState#value` 将会返回 `null`。
+如果需要更新状态，需要调用 `ValueState#update` 方法，直接更改 `ValueState#value` 的返回值可能不会被系统识别。
+容错处理将在 Flink 后台自动管理，你可以像与常规变量那样与状态变量进行交互。
-Below, you can see an example of how you can use a flag state to track potential fraudulent transactions.
+下边的示例，说明了如何使用标记状态来追踪可能的欺诈交易行为。
{% highlight java %}
- @Override
- public void processElement(
- Transaction transaction,
- Context context,
- Collector collector) throws Exception {
-
- // Get the current state for the current key
- Boolean lastTransactionWasSmall = flagState.value();
+@Override
+public void processElement(
+ Transaction transaction,
+ Context context,
+ Collector collector) throws Exception {
- // Check if the flag is set
- if (lastTransactionWasSmall != null) {
- if (transaction.getAmount() > LARGE_AMOUNT) {
- // Output an alert downstream
- Alert alert = new Alert();
- alert.setId(transaction.getAccountId());
+ // Get the current state for the current key
+ Boolean lastTransactionWasSmall = flagState.value();
- collector.collect(alert);
- }
+ // Check if the flag is set
+ if (lastTransactionWasSmall != null) {
+ if (transaction.getAmount() > LARGE_AMOUNT) {
+ // Output an alert downstream
+ Alert alert = new Alert();
+ alert.setId(transaction.getAccountId());
- // Clean up our state
- flagState.clear();
+ collector.collect(alert);
}
- if (transaction.getAmount() < SMALL_AMOUNT) {
- // Set the flag to true
- flagState.update(true);
- }
+ // Clean up our state
+ flagState.clear();
+ }
+
+ if (transaction.getAmount() < SMALL_AMOUNT) {
+ // Set the flag to true
+ flagState.update(true);
}
+}
{% endhighlight %}
{% highlight scala %}
- override def processElement(
- transaction: Transaction,
- context: KeyedProcessFunction[Long, Transaction, Alert]#Context,
- collector: Collector[Alert]): Unit = {
-
- // Get the current state for the current key
- val lastTransactionWasSmall = flagState.value
-
- // Check if the flag is set
- if (lastTransactionWasSmall != null) {
- if (transaction.getAmount > FraudDetector.LARGE_AMOUNT) {
- // Output an alert downstream
- val alert = new Alert
- alert.setId(transaction.getAccountId)
-
- collector.collect(alert)
- }
- // Clean up our state
- flagState.clear()
+override def processElement(
+ transaction: Transaction,
+ context: KeyedProcessFunction[Long, Transaction, Alert]#Context,
+ collector: Collector[Alert]): Unit = {
+
+ // Get the current state for the current key
+ val lastTransactionWasSmall = flagState.value
+
+ // Check if the flag is set
+ if (lastTransactionWasSmall != null) {
+ if (transaction.getAmount > FraudDetector.LARGE_AMOUNT) {
+ // Output an alert downstream
+ val alert = new Alert
+ alert.setId(transaction.getAccountId)
+
+ collector.collect(alert)
}
+ // Clean up our state
+ flagState.clear()
+ }
- if (transaction.getAmount < FraudDetector.SMALL_AMOUNT) {
- // set the flag to true
- flagState.update(true)
- }
+ if (transaction.getAmount < FraudDetector.SMALL_AMOUNT) {
+ // set the flag to true
+ flagState.update(true)
}
+}
{% endhighlight %}
-For every transaction, the fraud detector checks the state of the flag for that account.
-Remember, `ValueState` is always scoped to the current key, i.e., account.
-If the flag is non-null, then the last transaction seen for that account was small, and so if the amount for this transaction is large, then the detector outputs a fraud alert.
+对于每笔交易，欺诈检测器都会检查该帐户的标记状态。
+请记住，`ValueState` 的作用域始终限于当前的 key，即信用卡帐户。
+如果标记状态不为空，则该帐户的上一笔交易是小额的，因此，如果当前这笔交易的金额很大，那么检测程序将输出报警信息。
-After that check, the flag state is unconditionally cleared.
-Either the current transaction caused a fraud alert, and the pattern is over, or the current transaction did not cause an alert, and the pattern is broken and needs to be restarted.
+在检查之后，不论是什么状态，都需要被清空。
+不管是当前交易触发了欺诈报警而造成模式的结束，还是当前交易没有触发报警而造成模式的中断，都需要重新开始新的模式检测。
-Finally, the transaction amount is checked to see if it is small.
-If so, then the flag is set so that it can be checked by the next event.
-Notice that `ValueState` actually has three states, unset ( `null`), `true`, and `false`, because all `ValueState`'s are nullable.
-This job only makes use of unset ( `null`) and `true` to check whether the flag is set or not.
+最后，检查当前交易的金额是否属于小额交易。
+如果是，那么需要设置标记状态，以便可以在下一个事件中对其进行检查。
+注意，`ValueState` 实际上有 3 种状态：unset (`null`)，`true`，和 `false`，`ValueState` 是允许空值的。
+我们的程序只使用了 unset (`null`) 和 `true` 两种来判断标记状态被设置了与否。
-## Fraud Detector v2: State + Time = ❤️
+## 欺诈检测器 v2：状态 + 时间 = ❤️
-Scammers don't wait long to make their large purchase to reduce the chances their test transaction is noticed.
-For example, suppose you wanted to set a 1 minute timeout to your fraud detector; i.e., in the previous example transactions 3 and 4 would only be considered fraud if they occurred within 1 minute of each other.
-Flink's `KeyedProcessFunction` allows you to set timers which invoke a callback method at some point in time in the future.
+骗子们在小额交易后不会等很久就进行大额消费，这样可以降低小额测试交易被发现的几率。
+比如，假设你为欺诈检测器设置了一分钟的超时，对于上边的例子，交易 3 和 交易 4 只有间隔在一分钟之内才被认为是欺诈交易。
+Flink 中的 `KeyedProcessFunction` 允许您设置计时器，该计时器在将来的某个时间点执行回调函数。
-Let's see how we can modify our Job to comply with our new requirements:
+让我们看看如何修改程序以符合我们的新要求：
-* Whenever the flag is set to `true`, also set a timer for 1 minute in the future.
-* When the timer fires, reset the flag by clearing its state.
-* If the flag is ever cleared the timer should be canceled.
+* 当标记状态被设置为 `true` 时，设置一个在当前时间一分钟后触发的定时器。
+* 当定时器被触发时，重置标记状态。
+* 当标记状态被重置时，删除定时器。
-To cancel a timer, you have to remember what time it is set for, and remembering implies state, so you will begin by creating a timer state along with your flag state.
+要删除一个定时器，你需要记录这个定时器的触发时间，这同样需要状态来实现，所以你需要在标记状态后也创建一个记录定时器时间的状态。
-`KeyedProcessFunction#processElement` is called with a `Context` that contains a timer service.
-The timer service can be used to query the current time, register timers, and delete timers.
-With this, you can set a timer for 1 minute in the future every time the flag is set and store the timestamp in `timerState`.
+`KeyedProcessFunction#processElement` 需要使用提供了定时器服务的 `Context` 来调用。
+定时器服务可以用于查询当前时间、注册定时器和删除定时器。
+使用它，你可以在标记状态被设置时，也设置一个当前时间一分钟后触发的定时器，同时，将触发时间保存到 `timerState` 状态中。
@@ -664,10 +685,10 @@ if (transaction.getAmount < FraudDetector.SMALL_AMOUNT) {
-Processing time is wall clock time, and is determined by the system clock of the machine running the operator.
+处理时间是本地时钟时间，这是由运行任务的服务器的系统时间来决定的。
-When a timer fires, it calls `KeyedProcessFunction#onTimer`.
-Overriding this method is how you can implement your callback to reset the flag.
+当定时器触发时，将会调用 `KeyedProcessFunction#onTimer` 方法。
+通过重写这个方法来实现一个你自己的重置状态的回调逻辑。
@@ -694,8 +715,8 @@ override def onTimer(
-Finally, to cancel the timer, you need to delete the registered timer and delete the timer state.
-You can wrap this in a helper method and call this method instead of `flagState.clear()`.
+最后，如果要取消定时器，你需要删除已经注册的定时器，并同时清空保存定时器的状态。
+你可以把这些逻辑封装到一个助手函数中，而不是直接调用 `flagState.clear()`。
-### Expected Output
+### 期望的结果
-Running this code with the provided `TransactionSource` will emit fraud alerts for account 3.
-You should see the following output in your task manager logs:
+使用已准备好的 `TransactionSource` 数据源运行这个代码，将会检测到账户 3 的欺诈行为，并输出报警信息。
+你将能够在你的 task manager 的日志中看到下边输出：
{% highlight bash %}
2019-08-19 14:22:06,220 INFO org.apache.flink.walkthrough.common.sink.AlertSink - Alert{id=3}
diff --git a/docs/getting-started/walkthroughs/index.md b/docs/getting-started/walkthroughs/index.md
index 02b09ee4ceed901fb68ef4959e4fffa023fd1861..055f1b85050c4eb3dbc847f5800da7a3aa180d3f 100644
--- a/docs/getting-started/walkthroughs/index.md
+++ b/docs/getting-started/walkthroughs/index.md
@@ -1,9 +1,9 @@
---
title: "Code Walkthroughs"
nav-id: walkthroughs
-nav-title: ' Code Walkthroughs'
+nav-title: 'Code Walkthroughs'
nav-parent_id: getting-started
-nav-pos: 10
+nav-pos: 20
---
+This walkthrough will quickly get you started building a pure Python Flink project.
+
+Note Python 3.5 or higher is required to run PyFlink. Run the following command to confirm that the command “python” in current environment points to Python 3.5+:
+
+{% highlight bash %}
+$ python --version
+# the version printed here must be 3.5+
+{% endhighlight %}
+
* This will be replaced by the TOC
{:toc}
-In this guide we will start from scratch and go from setting up a Flink Python project
-to running a Python Table API program.
-
## Setting up a Python Project
-Firstly, you can fire up your favorite IDE and create a Python project and then
-you need to install the PyFlink package. Please
-see [Build PyFlink]({{ site.baseurl }}/flinkDev/building.html#build-pyflink)
-for more details about this.
+You can begin by creating a Python project and installing the PyFlink package.
+PyFlink is available via PyPi and can be easily installed using `pip`.
+
+{% highlight bash %}
+$ python -m pip install apache-flink
+{% endhighlight %}
+
+You can also build PyFlink from source by following the [development guide]({{ site.baseurl }}/flinkDev/building.html#build-pyflink).
## Writing a Flink Python Table API Program
-The first step in a Flink Python Table API program is to create a `BatchTableEnvironment`
-(or `StreamTableEnvironment` if you are writing a streaming job). It is the main entry point
-for Python Table API jobs.
+Table API applications begin by declaring a table environment; either a `BatchTableEvironment` for batch applications or `StreamTableEnvironment` for streaming applications.
+This serves as the main entry point for interacting with the Flink runtime.
+It can be used for setting execution parameters such as restart strategy, default parallelism, etc.
+The table config allows setting Table API specific configurations.
{% highlight python %}
exec_env = ExecutionEnvironment.get_execution_environment()
@@ -49,22 +60,15 @@ t_config = TableConfig()
t_env = BatchTableEnvironment.create(exec_env, t_config)
{% endhighlight %}
-The `ExecutionEnvironment` (or `StreamExecutionEnvironment` if you are writing a streaming job)
-can be used to set execution parameters, such as the restart strategy, default parallelism, etc.
-
-The `TableConfig` can be used by setting the parameters such as the built-in catalog name, the
-threshold where generating code, etc.
-
-Next we will create a source table and a sink table.
+The the table environment created, you can declare source and sink tables.
{% highlight python %}
t_env.connect(FileSystem().path('/tmp/input')) \
.with_format(OldCsv()
- .line_delimiter(' ')
.field('word', DataTypes.STRING())) \
.with_schema(Schema()
.field('word', DataTypes.STRING())) \
- .register_table_source('mySource')
+ .create_temporary_table('mySource')
t_env.connect(FileSystem().path('/tmp/output')) \
.with_format(OldCsv()
@@ -74,34 +78,31 @@ t_env.connect(FileSystem().path('/tmp/output')) \
.with_schema(Schema()
.field('word', DataTypes.STRING())
.field('count', DataTypes.BIGINT())) \
- .register_table_sink('mySink')
+ .create_temporary_table('mySink')
{% endhighlight %}
-This registers a table named `mySource` and a table named `mySink` in the
-`ExecutionEnvironment`. The table `mySource` has only one column: word.
-It represents the words read from file `/tmp/input`. The table `mySink` has two columns:
-word and count. It writes data to file `/tmp/output`, with `\t` as the field delimiter.
+This registers a table named `mySource` and a table named `mySink` in the execution environment.
+The table `mySource` has only one column, word, and it consumes strings read from file `/tmp/input`.
+The table `mySink` has two columns, word and count, and writes data to the file `/tmp/output`, with `\t` as the field delimiter.
-Then we need to create a job which reads input from table `mySource`, preforms some
-operations and writes the results to table `mySink`.
+You can now create a job which reads input from table `mySource`, preforms some transformations, and writes the results to table `mySink`.
{% highlight python %}
-t_env.scan('mySource') \
+t_env.from_path('mySource') \
.group_by('word') \
.select('word, count(1)') \
.insert_into('mySink')
{% endhighlight %}
-The last thing is to start the actual Flink Python Table API job. All operations, such as
-creating sources, transformations and sinks only build up a graph of internal operations.
-Only when `t_env.execute(job_name)` is called, this graph of operations will be thrown on a cluster or
-executed on your local machine.
+Finally you must execute the actual Flink Python Table API job.
+All operations, such as creating sources, transformations and sinks are lazy.
+Only when `t_env.execute(job_name)` is called will the job be run.
{% highlight python %}
t_env.execute("tutorial_job")
{% endhighlight %}
-The complete code so far is as follows:
+The complete code so far:
{% highlight python %}
from pyflink.dataset import ExecutionEnvironment
@@ -115,11 +116,10 @@ t_env = BatchTableEnvironment.create(exec_env, t_config)
t_env.connect(FileSystem().path('/tmp/input')) \
.with_format(OldCsv()
- .line_delimiter(' ')
.field('word', DataTypes.STRING())) \
.with_schema(Schema()
.field('word', DataTypes.STRING())) \
- .register_table_source('mySource')
+ .create_temporary_table('mySource')
t_env.connect(FileSystem().path('/tmp/output')) \
.with_format(OldCsv()
@@ -129,9 +129,9 @@ t_env.connect(FileSystem().path('/tmp/output')) \
.with_schema(Schema()
.field('word', DataTypes.STRING())
.field('count', DataTypes.BIGINT())) \
- .register_table_sink('mySink')
+ .create_temporary_table('mySink')
-t_env.scan('mySource') \
+t_env.from_path('mySource') \
.group_by('word') \
.select('word, count(1)') \
.insert_into('mySink')
@@ -140,9 +140,13 @@ t_env.execute("tutorial_job")
{% endhighlight %}
## Executing a Flink Python Table API Program
+Firstly, you need to prepare input data in the "/tmp/input" file. You can choose the following command line to prepare the input data:
+
+{% highlight bash %}
+$ echo "flink\npyflink\nflink" > /tmp/input
+{% endhighlight %}
-You can run this example in your IDE or on the command line (suppose the job script file is
-WordCount.py):
+Next, you can run this example on the command line (Note: if the result file "/tmp/output" has already existed, you need to remove the file before running the example):
{% highlight bash %}
$ python WordCount.py
@@ -153,6 +157,14 @@ You can also submit the Python Table API program to a remote cluster, you can re
[Job Submission Examples]({{ site.baseurl }}/ops/cli.html#job-submission-examples)
for more details.
+Finally, you can see the execution result on the command line:
+
+{% highlight bash %}
+$ cat /tmp/output
+flink 2
+pyflink 1
+{% endhighlight %}
+
This should get you started with writing your own Flink Python Table API programs.
To learn more about the Python Table API, you can refer
[Flink Python Table API Docs]({{ site.pythondocs_baseurl }}/api/python) for more details.
diff --git a/docs/tutorials/python_table_api.zh.md b/docs/getting-started/walkthroughs/python_table_api.zh.md
similarity index 78%
rename from docs/tutorials/python_table_api.zh.md
rename to docs/getting-started/walkthroughs/python_table_api.zh.md
index bab36e16960354e22f978eacf6b3950d5c1cd324..0805cbe69bf55c2338a330595ba9fcddb7272fdb 100644
--- a/docs/tutorials/python_table_api.zh.md
+++ b/docs/getting-started/walkthroughs/python_table_api.zh.md
@@ -1,7 +1,7 @@
---
title: "Python API 教程"
nav-title: Python API
-nav-parent_id: apitutorials
+nav-parent_id: walkthroughs
nav-pos: 10
---
-Apache Flink offers a Table API as a unified, relational API for batch and stream processing, i.e., queries are executed with the same semantics on unbounded, real-time streams or bounded, batch data sets and produce the same results.
-The Table API in Flink is commonly used to ease the definition of data analytics, data pipelining, and ETL applications.
-
+Apache Filnk 提供 Table API 作为批处理和流处理统一的关系型API，
+即查询在无界实时流或有界批数据集上以相同的语义执行，并产生相同的结果。
+Flink 中的 Table API 通常用于简化数据分析，数据流水线和 ETL 应用程序的定义。
* This will be replaced by the TOC
{:toc}
-## What Will You Be Building?
-
-In this tutorial, you will learn how to build a continuous ETL pipeline for tracking financial transactions by account over time.
-You will start by building your report as a nightly batch job, and then migrate to a streaming pipeline.
+## 接下来你会构建什么？
-## Prerequisites
+在本教程中，你将学习如何构建连续的 ETL 流水线，以便按账户随时跟踪金融交易。
+首先你将报表构建为每晚执行的批处理作业，然后迁移到流式管道。
-This walkthrough assumes that you have some familiarity with Java or Scala, but you should be able to follow along even if you are coming from a different programming language.
-It also assumes that you are familiar with basic relational concepts such as `SELECT` and `GROUP BY` clauses.
+## 先决条件
-## Help, I’m Stuck!
+本演练假设你对 Java 和 Scala 有一定的了解，但即便你使用其他编程语言，相信也可以学会。
+它还假定你熟悉基本的关系概念比如 `SELECT` 和 `GROUP BY` 子句。
-If you get stuck, check out the [community support resources](https://flink.apache.org/community.html).
-In particular, Apache Flink's [user mailing list](https://flink.apache.org/community.html#mailing-lists) is consistently ranked as one of the most active of any Apache project and a great way to get help quickly.
+## 救命，我被困住了！
-## How To Follow Along
+如果你被难题困住了，可以在[社区](https://flink.apache.org/community.html)寻求帮助。
+值得一提的是，Apache Flink 的[用户邮件列表](https://flink.apache.org/community.html#mailing-lists)一直是最活跃的 Apache 项目之一，也是一个快速获得帮助的好途径。
-If you want to follow along, you will require a computer with:
+## 如何跟进
-* Java 8
+如果想要继续，你的电脑需要安装：
+* Java 8 or 11
* Maven
-A provided Flink Maven Archetype will create a skeleton project with all the necessary dependencies quickly:
+现成的 Flink Maven Archetype 可以快速创建一个具有所有必要依赖的框架项目：
- Note: For Maven 3.0 or higher, it is no longer possible to specify the repository (-DarchetypeCatalog) via the commandline. If you wish to use the snapshot repository, you need to add a repository entry to your settings.xml. For details about this change, please refer to Maven's official document
+ 注意：Maven 3.0 及更高版本，不再支持通过命令行指定仓库（-DarchetypeCatalog）。有关这个改动的详细信息，
+ 请参阅 Maven 官方文档
+ 如果你希望使用快照仓库，则需要在 settings.xml 文件中添加一个仓库条目。例如：
+{% highlight bash %}
+
+
+ apache
+
+
+
+ apache
+
+
+ apache-snapshots
+ https://repository.apache.org/content/repositories/snapshots/
+
+
+
+
+
+{% endhighlight %}
{% endunless %}
-You can edit the `groupId`, `artifactId` and `package` if you like. With the above parameters,
-Maven will create a project with all the dependencies to complete this tutorial.
-After importing the project into your editor, you will see a file with the following code which you can run directly inside your IDE.
+你可以根据自己的意愿修改 `groupId`、`artifactId` 和 `package` 参数。通过使用以上参数，
+Maven 将创建一个拥有全部所需依赖的项目来完成本教程。
+把项目导入编辑器后，你会看到一个包含以下代码的文件，你可以在 IDE 中直接运行它。
@@ -132,14 +151,14 @@ env.execute("Spend Report")
-## Breaking Down The Code
+## 代码详解
-#### The Execution Environment
+#### 运行环境
-The first two lines set up your `ExecutionEnvironment`.
-The execution environment is how you can set properties for your Job, specify whether you are writing a batch or a streaming application, and create your sources.
-This walkthrough begins with the batch environment since you are building a periodic batch report.
-It is then wrapped in a `BatchTableEnvironment` to have full access to the Table API.
+前两行设置了你的 `ExecutionEnvironment`。
+运行环境用来设置作业的属性、指定应用是批处理还是流处理，以及创建数据源。
+由于你正在建立一个定时的批处理报告，本教程以批处理环境作为开始。
+然后将其包装进 `BatchTableEnvironment` 中从而能够使用所有的 Tabel API。
@@ -158,12 +177,12 @@ val tEnv = BatchTableEnvironment.create(env)
-#### Registering Tables
+#### 注册表
-Next, tables are registered in the execution environment that you can use to connect to external systems for reading and writing both batch and streaming data.
-A table source provides access to data stored in external systems; such as a database, a key-value store, a message queue, or a file system.
-A table sink emits a table to an external storage system.
-Depending on the type of source and sink, they support different formats such as CSV, JSON, Avro, or Parquet.
+接下来，表将会被注册到运行环境之中，这样你就可以用它们连接外部系统以读取或写入批数据或流数据。
+source 提供对存储在外部系统中的数据的访问；例如数据库、键-值存储、消息队列或文件系统。
+sink 则将表中的数据发送到外部存储系统。
+根据 source 或 sink 的类型，它们支持不同的格式，如 CSV、JSON、Avro 或 Parquet。
@@ -181,16 +200,16 @@ tEnv.registerTableSink("spend_report", new SpendReportTableSink)
-Two tables are registered; a transaction input table, and a spend report output table.
-The transactions (`transactions`) table lets us read credit card transactions, which contain account ID's (`accountId`), timestamps (`timestamp`), and US$ amounts (`amount`).
-In this tutorial, the table is backed by data generated in memory to avoid any dependencies on external systems.
-In practice, the `BoundedTransactionTableSource` may be backed by a filesystem, a database, or any other static source.
-The spend report (`spend_report`) table logs each row with log level **INFO**, instead of writing to persistent storage, so you can easily see your results.
+上例代码注册了两张表。交易表作为输入表，支出报告表作为输出表。
+我们可以从交易（`transactions`）表中读取信用卡的交易记录，其中包含了账户 ID（`accountId`）字段、时间戳（`timestamp`）字段和交易金额（`amount`）字段。
+本教程中，该表使用内存中的数据，以避免对外部系统的任何依赖。
+而在实际情况下，`BoundedTransactionTableSource` 可能来源于文件系统、数据库或任何静态数据源。
+支出报告表 `spend_report` 用 **INFO** 日志级别将表的每一行数据记录到日志，而不是写入持久化存储，所以你可以很容易地查看结果。
-#### Registering A UDF
+#### 注册 UDF
-Along with the tables, a [user-defined function]({{ site.baseurl }}/dev/table/udfs.html) is registered for working with timestamps.
-This function takes a timestamp and rounds it down to the nearest hour.
+一个用来处理时间戳的[自定义函数]({{ site.baseurl }}/zh/dev/table/functions/udfs.html)随表一起被注册到tEnv中。
+此函数将时间戳向下舍入到最接近的小时。
@@ -206,10 +225,10 @@ val truncateDateToHour = new TruncateDateToHour
-#### The Query
+#### 查询
-With the environment configured and tables registered, you are ready to build your first application.
-From the `TableEnvironment` you can `scan` an input table to read its rows and then write those results into an output table using `insertInto`.
+完成配置环境和注册表后，你已准备好构建第一个应用程序。
+从 `TableEnvironment` 中，你可以 `scan` 一个输入表读取其中的行，然后用 `insertInto` 把这些数据写到输出表中。
@@ -229,12 +248,13 @@ tEnv
-Initially, the Job reads all transactions and logs them out with log level **INFO**.
+最初，作业读取所有的交易记录并用 **INFO** 日志级别将其记录下来。
+
+#### 运行
-#### Execute
+Flink 应用是延迟构建的，只有完全定义好之后才交付集群运行。
+你可以调用 `ExecutionEnvironment#execute` 来开始作业的执行并给它取一个名字。
-Flink applications are built lazily and shipped to the cluster for execution only once fully formed.
-You call `ExecutionEnvironment#execute` to begin the execution of your Job by giving it a name.
@@ -250,14 +270,13 @@ env.execute("Spend Report")
-## Attempt One
-
-Now with the skeleton of a Job set-up, you are ready to add some business logic.
-The goal is to build a report that shows the total spend for each account across each hour of the day.
-Just like a SQL query, Flink can select the required fields and group by your keys.
-Because the timestamp field has millisecond granularity, you can use the UDF to round it down to the nearest hour.
-Finally, select all the fields, summing the total spend per account-hour pair with the built-in `sum` [aggregate function]({{ site.baseurl }}/dev/table/functions.html#aggregate-functions).
+## 尝试一下
+现在有了作业设置的框架，你就可以添加一些业务逻辑了。
+目标是建立一个报表来显示每天每小时每个账户的总支出。
+就像一个 SQL 查询一样，Flink 可以选取所需的字段并且按键分组。
+由于时间戳字段具有毫秒的粒度，你可以使用自定义函数将其舍入到最近的小时。
+最后，选取所有的字段，用内建的 `sum` [聚合函数]({{ site.baseurl }}/zh/dev/table/functions/systemFunctions.html#aggregate-functions)函数合计每一个账户每小时的支出。
{% highlight java %}
@@ -282,10 +301,9 @@ tEnv
-This query consumes all records from the `transactions` table, calculates the report, and outputs the results in an efficient, scalable manner.
-
+这个查询处理了 `transactions` 表中的所有记录，计算报告，并以高效、可扩展的方式输出结果。
{% highlight raw %}
-# Query 1 output showing account id, timestamp, and amount
+# 查询 1 的输出显示了账户 id、时间戳和消费总额。
> 1, 2019-01-01 00:00:00.0, $567.87
> 2, 2019-01-01 00:00:00.0, $726.23
@@ -299,11 +317,11 @@ This query consumes all records from the `transactions` table, calculates the re
> 2, 2019-01-01 04:00:00.0, $760.76
{% endhighlight %}
-## Adding Windows
+## 添加窗口
-Grouping data based on time is a typical operation in data processing, especially when working with infinite streams.
-A grouping based on time is called a [window]({{ site.baseurl }} /dev/stream/operators/windows.html) and Flink offers flexible windowing semantics.
-The most basic type of window is called a `Tumble` window, which has a fixed size and whose buckets do not overlap.
+根据时间进行分组在数据处理中是一种很常见的方式，特别是在处理无限的数据流时。
+基于时间的分组称为[窗口]({{ site.baseurl }}/zh/dev/stream/operators/windows.html) ，Flink 提供了灵活的窗口语义。
+其中最基础的是 `Tumble` window （滚动窗口），它具有固定大小且窗口之间不重叠。
@@ -329,16 +347,16 @@ tEnv
-This defines your application as using one hour tumbling windows based on the timestamp column.
-So a row with timestamp `2019-06-01 01:23:47` is put in the `2019-06-01 01:00:00` window.
+你的应用将会使用基于时间戳字段的一小时的滚动窗口。
+因此时间戳是 `2019-06-01 01:23:47` 的行被放入 `2019-06-01 01:00:00` 这个时间窗口之中。
-Aggregations based on time are unique because time, as opposed to other attributes, generally moves forward in a continuous streaming application.
-In a batch context, windows offer a convenient API for grouping records by a timestamp attribute.
+在持续的流式应用中，基于时间的聚合结果是唯一的，因为相较于其他属性，时间通常会向前移动。
+在批处理环境中，窗口提供了一个方便的 API，用于按时间戳属性对记录进行分组。
-Running the updated query will produce identical results as before.
+运行这个更新过的查询将会得到和之前一样的结果。
{% highlight raw %}
-# Query 2 output showing account id, timestamp, and amount
+# 查询 2 的输出显示了账户 id、时间戳和消费总额
> 1, 2019-01-01 00:00:00.0, $567.87
> 2, 2019-01-01 00:00:00.0, $726.23
@@ -352,13 +370,13 @@ Running the updated query will produce identical results as before.
> 2, 2019-01-01 04:00:00.0, $760.76
{% endhighlight %}
-## Once More, With Streaming!
+## 通过流处理的方式再来一次！
-Because Flink's Table API offers consistent syntax and semantics for both batch and streaming, migrating from one to the other requires just two steps.
+因为 Flink 的 Table API 为批处理和流处理提供了相同的语法和语义，从一种方式迁移到另一种方式只需要两步。
-The first step is to replace the batch `ExecutionEnvironment` with its streaming counterpart, `StreamExecutionEnvironment`, which creates a continuous streaming Job.
-It includes stream-specific configurations, such as the time characteristic, which when set to [event time]({{ site.baseurl }}/dev/event_time.html) guarantees consistent results even when faced with out-of-order events or a Job failure.
-This is what will be used by your `Tumble` window when grouping records.
+第一步是把批处理的 `ExecutionEnvironment` 替换成流处理对应的 `StreamExecutionEnvironment`，后者创建连续的流作业。
+它包含特定于流处理的配置，比如时间特性。当这个属性被设置成 [事件时间]({{ site.baseurl }}/zh/dev/event_time.html)时，它能保证即使遭遇乱序事件或者作业失败的情况也能输出一致的结果。
+滚动窗口在对数据进行分组时就运用了这个特性。
@@ -380,10 +398,10 @@ val tEnv = StreamTableEnvironment.create(env)
-The second step is to migrate from a bounded data source to an infinite data source.
-The project comes with an `UnboundedTransactionTableSource` that continuously creates transaction events in real-time.
-Similar to the `BoundedTransactionTableSource` this table is backed by data generated in memory to avoid any dependencies on external systems.
-In practice, this table might read from a streaming source such as Apache Kafka, AWS Kinesis, or Pravega.
+第二步就是把有界的数据源替换成无限的数据源。
+这个项目通过 `UnboundedTransactionTableSource` 持续不断地实时生成交易事件。
+与 `BoundedTransactionTableSource` 一样，这个表也是通过在内存中生成数据从而不依赖外部系统。
+在实践中，这个表可能从一个流式数据源中读取数据，比如 Apache Kafka、AWS Kinesis 或者 Pravega。
@@ -399,27 +417,29 @@ tEnv.registerTableSource("transactions", new UnboundedTransactionTableSource)
-And that's it, a fully functional, stateful, distributed streaming application!
-The query continuously consumes the stream of transactions, computes the hourly spendings, and emits results as soon as they are ready.
-Since the input is unbounded, the query keeps running until it is manually stopped.
-And because the Job uses time window-based aggregations, Flink can perform specific optimizations such as state clean up when the framework knows that no more records will arrive for a particular window.
+这就是一个功能齐全、有状态的分布式流式应用！
+这个查询会持续处理交易流，计算每小时的消费额，然后实时输出结果。
+由于输入是无界的，因此查询将一直运行，直到手动停止为止。
+因为这个作业使用了基于时间窗口的聚合，Flink 可以使用一些特定的优化，比如当系统知道一个特定的窗口不会再有新的数据到来，它就会对状态进行清理。
{% highlight raw %}
-# Query 3 output showing account id, timestamp, and amount
+# 查询 3 的输出显示了账户 id、时间戳消费总额
-# These rows are calculated continuously over the hour
-# and output immediately at the end of the hour
> 1, 2019-01-01 00:00:00.0, $567.87
> 2, 2019-01-01 00:00:00.0, $726.23
-# Flink begins computing these rows as soon as
-# as the first record for the window arrives
+# 这些行是在这一小时中连续计算的
+# 并在这一小时结束时立刻输出
+
> 1, 2019-01-01 01:00:00.0, $686.87
> 2, 2019-01-01 01:00:00.0, $810.06
+# 当接收到该窗口的第一条数据时
+# Flink 就开始计算了
+
{% endhighlight %}
-## Final Application
+## 最终程序
diff --git a/docs/index.md b/docs/index.md
index 9dddcf3b6b7a41fc26101bfc433e2831db1df36d..9837395b56c9c0046bc5652c3d0786a84aebc485 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -36,9 +36,6 @@ Apache Flink is an open source platform for distributed stream and batch data pr
* **Docker Playgrounds**: Set up a sandboxed Flink environment in just a few minutes to explore and play with Flink.
* [Run and manage Flink streaming applications](./getting-started/docker-playgrounds/flink-operations-playground.html)
-* **Tutorials**: Install Flink on your local machine.
- * [Setup a local Flink cluster](./getting-started/tutorials/local_setup.html)
-
* **Concepts**: Learn about Flink's basic concepts to better understand the documentation.
* [Dataflow Programming Model](concepts/programming-model.html)
* [Distributed Runtime](concepts/runtime.html)
@@ -61,6 +58,7 @@ Before putting your Flink job into production, read the [Production Readiness Ch
Release notes cover important changes between Flink versions. Please carefully read these notes if you plan to upgrade your Flink setup to a later version.
+* [Release notes for Flink 1.10](release-notes/flink-1.10.html).
* [Release notes for Flink 1.9](release-notes/flink-1.9.html).
* [Release notes for Flink 1.8](release-notes/flink-1.8.html).
* [Release notes for Flink 1.7](release-notes/flink-1.7.html).
diff --git a/docs/index.zh.md b/docs/index.zh.md
index 94a39df01a7b2485a1381ddcde9501f965d1c5e1..2315024829eabee159379878946ad6763cd598a1 100644
--- a/docs/index.zh.md
+++ b/docs/index.zh.md
@@ -37,9 +37,6 @@ Apache Flink 是一个分布式流批一体化的开源平台。Flink 的核心
* **Docker 游乐场**: 你只需花几分钟搭建 Flink 沙盒环境，就可以探索和使用 Flink 了。
* [运行与管理 Flink 流处理应用](./getting-started/docker-playgrounds/flink-operations-playground.html)
-* **教程**: 在你的本地机器上安装 Flink。
- * [搭建本地 Flink 集群](./getting-started/tutorials/local_setup.html)
-
* **概念**: 学习 Flink 的基本概念能更好地理解文档。
* [数据流编程模型](concepts/programming-model.html)
* [分布式执行](concepts/runtime.html)
@@ -62,6 +59,7 @@ API 参考列举并解释了 Flink API 的所有功能。
发布日志包含了 Flink 版本之间的重大更新。请在你升级 Flink 之前仔细阅读相应的发布日志。
+* [Flink 1.10 的发布日志](release-notes/flink-1.10.html).
* [Flink 1.9 的发布日志](release-notes/flink-1.9.html)。
* [Flink 1.8 的发布日志](release-notes/flink-1.8.html)。
* [Flink 1.7 的发布日志](release-notes/flink-1.7.html)。
diff --git a/docs/internals/components.md b/docs/internals/components.md
deleted file mode 100644
index 4d746107f9b2bff6d8636c50fe02d4d451494f02..0000000000000000000000000000000000000000
--- a/docs/internals/components.md
+++ /dev/null
@@ -1,60 +0,0 @@
----
-title: "Component Stack"
-nav-parent_id: internals
-nav-pos: 1
----
-
-
-As a software stack, Flink is a layered system. The different layers of the stack build on
-top of each other and raise the abstraction level of the program representations they accept:
-
-- The **runtime** layer receives a program in the form of a *JobGraph*. A JobGraph is a generic parallel
-data flow with arbitrary tasks that consume and produce data streams.
-
-- Both the **DataStream API** and the **DataSet API** generate JobGraphs through separate compilation
-processes. The DataSet API uses an optimizer to determine the optimal plan for the program, while
-the DataStream API uses a stream builder.
-
-- The JobGraph is executed according to a variety of deployment options available in Flink (e.g., local,
-remote, YARN, etc)
-
-- Libraries and APIs that are bundled with Flink generate DataSet or DataStream API programs. These are
-Table for queries on logical tables, complex event processing, and Gelly for graph processing.
-
-You can click on the components in the figure to learn more.
-
-
-
-
-
-
-
-{% top %}
diff --git a/docs/internals/components.zh.md b/docs/internals/components.zh.md
deleted file mode 100644
index 0df3065c9011094c66ae015277f95c428348a3bf..0000000000000000000000000000000000000000
--- a/docs/internals/components.zh.md
+++ /dev/null
@@ -1,60 +0,0 @@
----
-title: "组件堆栈"
-nav-parent_id: internals
-nav-pos: 1
----
-
-
-As a software stack, Flink is a layered system. The different layers of the stack build on
-top of each other and raise the abstraction level of the program representations they accept:
-
-- The **runtime** layer receives a program in the form of a *JobGraph*. A JobGraph is a generic parallel
-data flow with arbitrary tasks that consume and produce data streams.
-
-- Both the **DataStream API** and the **DataSet API** generate JobGraphs through separate compilation
-processes. The DataSet API uses an optimizer to determine the optimal plan for the program, while
-the DataStream API uses a stream builder.
-
-- The JobGraph is executed according to a variety of deployment options available in Flink (e.g., local,
-remote, YARN, etc)
-
-- Libraries and APIs that are bundled with Flink generate DataSet or DataStream API programs. These are
-Table for queries on logical tables, complex event processing, and Gelly for graph processing.
-
-You can click on the components in the figure to learn more.
-
-
-
-
-
-
-
-{% top %}
diff --git a/docs/internals/stream_checkpointing.md b/docs/internals/stream_checkpointing.md
deleted file mode 100644
index 5b210787d0a48a80d9f03916e05346caf6914712..0000000000000000000000000000000000000000
--- a/docs/internals/stream_checkpointing.md
+++ /dev/null
@@ -1,173 +0,0 @@
----
-title: "Data Streaming Fault Tolerance"
-nav-title: Fault Tolerance for Data Streaming
-nav-parent_id: internals
-nav-pos: 3
----
-
-
-This document describes Flink's fault tolerance mechanism for streaming data flows.
-
-* This will be replaced by the TOC
-{:toc}
-
-
-## Introduction
-
-Apache Flink offers a fault tolerance mechanism to consistently recover the state of data streaming applications.
-The mechanism ensures that even in the presence of failures, the program's state will eventually reflect every
-record from the data stream **exactly once**. Note that there is a switch to *downgrade* the guarantees to *at least once*
-(described below).
-
-The fault tolerance mechanism continuously draws snapshots of the distributed streaming data flow. For streaming applications
-with small state, these snapshots are very light-weight and can be drawn frequently without much impact on performance.
-The state of the streaming applications is stored at a configurable place (such as the master node, or HDFS).
-
-In case of a program failure (due to machine-, network-, or software failure), Flink stops the distributed streaming dataflow.
-The system then restarts the operators and resets them to the latest successful checkpoint. The input streams are reset to the
-point of the state snapshot. Any records that are processed as part of the restarted parallel dataflow are guaranteed to not
-have been part of the previously checkpointed state.
-
-*Note:* By default, checkpointing is disabled. See [Checkpointing]({{ site.baseurl }}/dev/stream/state/checkpointing.html) for details on how to enable and configure checkpointing.
-
-*Note:* For this mechanism to realize its full guarantees, the data stream source (such as message queue or broker) needs to be able
-to rewind the stream to a defined recent point. [Apache Kafka](http://kafka.apache.org) has this ability and Flink's connector to
-Kafka exploits this ability. See [Fault Tolerance Guarantees of Data Sources and Sinks]({{ site.baseurl }}/dev/connectors/guarantees.html) for
-more information about the guarantees provided by Flink's connectors.
-
-*Note:* Because Flink's checkpoints are realized through distributed snapshots, we use the words *snapshot* and *checkpoint* interchangeably.
-
-
-## Checkpointing
-
-The central part of Flink's fault tolerance mechanism is drawing consistent snapshots of the distributed data stream and operator state.
-These snapshots act as consistent checkpoints to which the system can fall back in case of a failure. Flink's mechanism for drawing these
-snapshots is described in "[Lightweight Asynchronous Snapshots for Distributed Dataflows](http://arxiv.org/abs/1506.08603)". It is inspired by
-the standard [Chandy-Lamport algorithm](http://research.microsoft.com/en-us/um/people/lamport/pubs/chandy.pdf) for distributed snapshots and is
-specifically tailored to Flink's execution model.
-
-
-
-### Barriers
-
-A core element in Flink's distributed snapshotting are the *stream barriers*. These barriers are injected into the data stream and flow
-with the records as part of the data stream. Barriers never overtake records, they flow strictly in line.
-A barrier separates the records in the data stream into the set of records that goes into the
-current snapshot, and the records that go into the next snapshot. Each barrier carries the ID of the snapshot whose records it pushed in front
-of it. Barriers do not interrupt the flow of the stream and are hence very lightweight. Multiple barriers from different snapshots can be in
-the stream at the same time, which means that various snapshots may happen concurrently.
-
-
-
-
-
-Stream barriers are injected into the parallel data flow at the stream sources. The point where the barriers for snapshot *n* are injected
-(let's call it Sn) is the position in the source stream up to which the snapshot covers the data. For example, in Apache Kafka, this
-position would be the last record's offset in the partition. This position Sn is reported to the *checkpoint coordinator* (Flink's JobManager).
-
-The barriers then flow downstream. When an intermediate operator has received a barrier for snapshot *n* from all of its input streams, it emits a barrier
-for snapshot *n* into all of its outgoing streams. Once a sink operator (the end of a streaming DAG) has received the barrier *n* from all of its
-input streams, it acknowledges that snapshot *n* to the checkpoint coordinator. After all sinks have acknowledged a snapshot, it is considered completed.
-
-Once snapshot *n* has been completed, the job will never again ask the source for records from before Sn, since at that point these records (and
-their descendant records) will have passed through the entire data flow topology.
-
-
-
-
-
-Operators that receive more than one input stream need to *align* the input streams on the snapshot barriers. The figure above illustrates this:
-
- - As soon as the operator receives snapshot barrier *n* from an incoming stream, it cannot process any further records from that stream until it has received
-the barrier *n* from the other inputs as well. Otherwise, it would mix records that belong to snapshot *n* and with records that belong to snapshot *n+1*.
- - Streams that report barrier *n* are temporarily set aside. Records that are received from these streams are not processed, but put into an input buffer.
- - Once the last stream has received barrier *n*, the operator emits all pending outgoing records, and then emits snapshot *n* barriers itself.
- - After that, it resumes processing records from all input streams, processing records from the input buffers before processing the records from the streams.
-
-
-### State
-
-When operators contain any form of *state*, this state must be part of the snapshots as well. Operator state comes in different forms:
-
- - *User-defined state*: This is state that is created and modified directly by the transformation functions (like `map()` or `filter()`). See [State in Streaming Applications]({{ site.baseurl }}/dev/stream/state/index.html) for details.
- - *System state*: This state refers to data buffers that are part of the operator's computation. A typical example for this state are the *window buffers*, inside which the system collects (and aggregates) records for windows until the window is evaluated and evicted.
-
-Operators snapshot their state at the point in time when they have received all snapshot barriers from their input streams, and before emitting the barriers to their output streams. At that point, all updates to the state from records before the barriers will have been made, and no updates that depend on records from after the barriers have been applied. Because the state of a snapshot may be large, it is stored in a configurable *[state backend]({{ site.baseurl }}/ops/state/state_backends.html)*. By default, this is the JobManager's memory, but for production use a distributed reliable storage should be configured (such as HDFS). After the state has been stored, the operator acknowledges the checkpoint, emits the snapshot barrier into the output streams, and proceeds.
-
-The resulting snapshot now contains:
-
- - For each parallel stream data source, the offset/position in the stream when the snapshot was started
- - For each operator, a pointer to the state that was stored as part of the snapshot
-
-
-
-
-
-
-### Exactly Once vs. At Least Once
-
-The alignment step may add latency to the streaming program. Usually, this extra latency is on the order of a few milliseconds, but we have seen cases where the latency
-of some outliers increased noticeably. For applications that require consistently super low latencies (few milliseconds) for all records, Flink has a switch to skip the
-stream alignment during a checkpoint. Checkpoint snapshots are still drawn as soon as an operator has seen the checkpoint barrier from each input.
-
-When the alignment is skipped, an operator keeps processing all inputs, even after some checkpoint barriers for checkpoint *n* arrived. That way, the operator also processes
-elements that belong to checkpoint *n+1* before the state snapshot for checkpoint *n* was taken.
-On a restore, these records will occur as duplicates, because they are both included in the state snapshot of checkpoint *n*, and will be replayed as part
-of the data after checkpoint *n*.
-
-*NOTE*: Alignment happens only for operators with multiple predecessors (joins) as well as operators with multiple senders (after a stream repartitioning/shuffle).
-Because of that, dataflows with only embarrassingly parallel streaming operations (`map()`, `flatMap()`, `filter()`, ...) actually give *exactly once* guarantees even
-in *at least once* mode.
-
-
-### Asynchronous State Snapshots
-
-Note that the above described mechanism implies that operators stop processing input records while they are storing a snapshot of their state in the *state backend*. This *synchronous* state snapshot introduces a delay every time a snapshot is taken.
-
-It is possible to let an operator continue processing while it stores its state snapshot, effectively letting the state snapshots happen *asynchronously* in the background. To do that, the operator must be able to produce a state object that should be stored in a way such that further modifications to the operator state do not affect that state object. For example, *copy-on-write* data structures, such as are used in RocksDB, have this behavior.
-
-After receiving the checkpoint barriers on its inputs, the operator starts the asynchronous snapshot copying of its state. It immediately emits the barrier to its outputs and continues with the regular stream processing. Once the background copy process has completed, it acknowledges the checkpoint to the checkpoint coordinator (the JobManager). The checkpoint is now only complete after all sinks have received the barriers and all stateful operators have acknowledged their completed backup (which may be after the barriers reach the sinks).
-
-See [State Backends]({{ site.baseurl }}/ops/state/state_backends.html) for details on the state snapshots.
-
-
-## Recovery
-
-Recovery under this mechanism is straightforward: Upon a failure, Flink selects the latest completed checkpoint *k*. The system then re-deploys the
-entire distributed dataflow, and gives each operator the state that was snapshotted as part of checkpoint *k*. The sources are set to start reading the
-stream from position Sk. For example in Apache Kafka, that means telling the consumer to start fetching from offset Sk.
-
-If state was snapshotted incrementally, the operators start with the state of the latest full snapshot and then apply a series of incremental snapshot updates to that state.
-
-See [Restart Strategies]({{ site.baseurl }}/dev/restart_strategies.html) for more information.
-
-## Operator Snapshot Implementation
-
-When operator snapshots are taken, there are two parts: the **synchronous** and the **asynchronous** parts.
-
-Operators and state backends provide their snapshots as a Java `FutureTask`. That task contains the state where the *synchronous* part
-is completed and the *asynchronous* part is pending. The asynchronous part is then executed by a background thread for that checkpoint.
-
-Operators that checkpoint purely synchronously return an already completed `FutureTask`.
-If an asynchronous operation needs to be performed, it is executed in the `run()` method of that `FutureTask`.
-
-The tasks are cancelable, so that streams and other resource consuming handles can be released.
-
-{% top %}
diff --git a/docs/internals/stream_checkpointing.zh.md b/docs/internals/stream_checkpointing.zh.md
deleted file mode 100644
index 0774022326d348d4fbe8c556891226455b133c93..0000000000000000000000000000000000000000
--- a/docs/internals/stream_checkpointing.zh.md
+++ /dev/null
@@ -1,173 +0,0 @@
----
-title: "数据流容错"
-nav-title: 数据流容错
-nav-parent_id: internals
-nav-pos: 3
----
-
-
-This document describes Flink's fault tolerance mechanism for streaming data flows.
-
-* This will be replaced by the TOC
-{:toc}
-
-
-## Introduction
-
-Apache Flink offers a fault tolerance mechanism to consistently recover the state of data streaming applications.
-The mechanism ensures that even in the presence of failures, the program's state will eventually reflect every
-record from the data stream **exactly once**. Note that there is a switch to *downgrade* the guarantees to *at least once*
-(described below).
-
-The fault tolerance mechanism continuously draws snapshots of the distributed streaming data flow. For streaming applications
-with small state, these snapshots are very light-weight and can be drawn frequently without much impact on performance.
-The state of the streaming applications is stored at a configurable place (such as the master node, or HDFS).
-
-In case of a program failure (due to machine-, network-, or software failure), Flink stops the distributed streaming dataflow.
-The system then restarts the operators and resets them to the latest successful checkpoint. The input streams are reset to the
-point of the state snapshot. Any records that are processed as part of the restarted parallel dataflow are guaranteed to not
-have been part of the previously checkpointed state.
-
-*Note:* By default, checkpointing is disabled. See [Checkpointing]({{ site.baseurl }}/dev/stream/state/checkpointing.html) for details on how to enable and configure checkpointing.
-
-*Note:* For this mechanism to realize its full guarantees, the data stream source (such as message queue or broker) needs to be able
-to rewind the stream to a defined recent point. [Apache Kafka](http://kafka.apache.org) has this ability and Flink's connector to
-Kafka exploits this ability. See [Fault Tolerance Guarantees of Data Sources and Sinks]({{ site.baseurl }}/dev/connectors/guarantees.html) for
-more information about the guarantees provided by Flink's connectors.
-
-*Note:* Because Flink's checkpoints are realized through distributed snapshots, we use the words *snapshot* and *checkpoint* interchangeably.
-
-
-## Checkpointing
-
-The central part of Flink's fault tolerance mechanism is drawing consistent snapshots of the distributed data stream and operator state.
-These snapshots act as consistent checkpoints to which the system can fall back in case of a failure. Flink's mechanism for drawing these
-snapshots is described in "[Lightweight Asynchronous Snapshots for Distributed Dataflows](http://arxiv.org/abs/1506.08603)". It is inspired by
-the standard [Chandy-Lamport algorithm](http://research.microsoft.com/en-us/um/people/lamport/pubs/chandy.pdf) for distributed snapshots and is
-specifically tailored to Flink's execution model.
-
-
-
-### Barriers
-
-A core element in Flink's distributed snapshotting are the *stream barriers*. These barriers are injected into the data stream and flow
-with the records as part of the data stream. Barriers never overtake records, they flow strictly in line.
-A barrier separates the records in the data stream into the set of records that goes into the
-current snapshot, and the records that go into the next snapshot. Each barrier carries the ID of the snapshot whose records it pushed in front
-of it. Barriers do not interrupt the flow of the stream and are hence very lightweight. Multiple barriers from different snapshots can be in
-the stream at the same time, which means that various snapshots may happen concurrently.
-
-
-
-
-
-Stream barriers are injected into the parallel data flow at the stream sources. The point where the barriers for snapshot *n* are injected
-(let's call it Sn) is the position in the source stream up to which the snapshot covers the data. For example, in Apache Kafka, this
-position would be the last record's offset in the partition. This position Sn is reported to the *checkpoint coordinator* (Flink's JobManager).
-
-The barriers then flow downstream. When an intermediate operator has received a barrier for snapshot *n* from all of its input streams, it emits a barrier
-for snapshot *n* into all of its outgoing streams. Once a sink operator (the end of a streaming DAG) has received the barrier *n* from all of its
-input streams, it acknowledges that snapshot *n* to the checkpoint coordinator. After all sinks have acknowledged a snapshot, it is considered completed.
-
-Once snapshot *n* has been completed, the job will never again ask the source for records from before Sn, since at that point these records (and
-their descendant records) will have passed through the entire data flow topology.
-
-
-
-
-
-Operators that receive more than one input stream need to *align* the input streams on the snapshot barriers. The figure above illustrates this:
-
- - As soon as the operator receives snapshot barrier *n* from an incoming stream, it cannot process any further records from that stream until it has received
-the barrier *n* from the other inputs as well. Otherwise, it would mix records that belong to snapshot *n* and with records that belong to snapshot *n+1*.
- - Streams that report barrier *n* are temporarily set aside. Records that are received from these streams are not processed, but put into an input buffer.
- - Once the last stream has received barrier *n*, the operator emits all pending outgoing records, and then emits snapshot *n* barriers itself.
- - After that, it resumes processing records from all input streams, processing records from the input buffers before processing the records from the streams.
-
-
-### State
-
-When operators contain any form of *state*, this state must be part of the snapshots as well. Operator state comes in different forms:
-
- - *User-defined state*: This is state that is created and modified directly by the transformation functions (like `map()` or `filter()`). See [State in Streaming Applications]({{ site.baseurl }}/dev/stream/state/index.html) for details.
- - *System state*: This state refers to data buffers that are part of the operator's computation. A typical example for this state are the *window buffers*, inside which the system collects (and aggregates) records for windows until the window is evaluated and evicted.
-
-Operators snapshot their state at the point in time when they have received all snapshot barriers from their input streams, and before emitting the barriers to their output streams. At that point, all updates to the state from records before the barriers will have been made, and no updates that depend on records from after the barriers have been applied. Because the state of a snapshot may be large, it is stored in a configurable *[state backend]({{ site.baseurl }}/ops/state/state_backends.html)*. By default, this is the JobManager's memory, but for production use a distributed reliable storage should be configured (such as HDFS). After the state has been stored, the operator acknowledges the checkpoint, emits the snapshot barrier into the output streams, and proceeds.
-
-The resulting snapshot now contains:
-
- - For each parallel stream data source, the offset/position in the stream when the snapshot was started
- - For each operator, a pointer to the state that was stored as part of the snapshot
-
-
-
-
-
-
-### Exactly Once vs. At Least Once
-
-The alignment step may add latency to the streaming program. Usually, this extra latency is on the order of a few milliseconds, but we have seen cases where the latency
-of some outliers increased noticeably. For applications that require consistently super low latencies (few milliseconds) for all records, Flink has a switch to skip the
-stream alignment during a checkpoint. Checkpoint snapshots are still drawn as soon as an operator has seen the checkpoint barrier from each input.
-
-When the alignment is skipped, an operator keeps processing all inputs, even after some checkpoint barriers for checkpoint *n* arrived. That way, the operator also processes
-elements that belong to checkpoint *n+1* before the state snapshot for checkpoint *n* was taken.
-On a restore, these records will occur as duplicates, because they are both included in the state snapshot of checkpoint *n*, and will be replayed as part
-of the data after checkpoint *n*.
-
-*NOTE*: Alignment happens only for operators with multiple predecessors (joins) as well as operators with multiple senders (after a stream repartitioning/shuffle).
-Because of that, dataflows with only embarrassingly parallel streaming operations (`map()`, `flatMap()`, `filter()`, ...) actually give *exactly once* guarantees even
-in *at least once* mode.
-
-
-### Asynchronous State Snapshots
-
-Note that the above described mechanism implies that operators stop processing input records while they are storing a snapshot of their state in the *state backend*. This *synchronous* state snapshot introduces a delay every time a snapshot is taken.
-
-It is possible to let an operator continue processing while it stores its state snapshot, effectively letting the state snapshots happen *asynchronously* in the background. To do that, the operator must be able to produce a state object that should be stored in a way such that further modifications to the operator state do not affect that state object. For example, *copy-on-write* data structures, such as are used in RocksDB, have this behavior.
-
-After receiving the checkpoint barriers on its inputs, the operator starts the asynchronous snapshot copying of its state. It immediately emits the barrier to its outputs and continues with the regular stream processing. Once the background copy process has completed, it acknowledges the checkpoint to the checkpoint coordinator (the JobManager). The checkpoint is now only complete after all sinks have received the barriers and all stateful operators have acknowledged their completed backup (which may be after the barriers reach the sinks).
-
-See [State Backends]({{ site.baseurl }}/ops/state/state_backends.html) for details on the state snapshots.
-
-
-## Recovery
-
-Recovery under this mechanism is straightforward: Upon a failure, Flink selects the latest completed checkpoint *k*. The system then re-deploys the
-entire distributed dataflow, and gives each operator the state that was snapshotted as part of checkpoint *k*. The sources are set to start reading the
-stream from position Sk. For example in Apache Kafka, that means telling the consumer to start fetching from offset Sk.
-
-If state was snapshotted incrementally, the operators start with the state of the latest full snapshot and then apply a series of incremental snapshot updates to that state.
-
-See [Restart Strategies]({{ site.baseurl }}/dev/restart_strategies.html) for more information.
-
-## Operator Snapshot Implementation
-
-When operator snapshots are taken, there are two parts: the **synchronous** and the **asynchronous** parts.
-
-Operators and state backends provide their snapshots as a Java `FutureTask`. That task contains the state where the *synchronous* part
-is completed and the *asynchronous* part is pending. The asynchronous part is then executed by a background thread for that checkpoint.
-
-Operators that checkpoint purely synchronously return an already completed `FutureTask`.
-If an asynchronous operation needs to be performed, it is executed in the `run()` method of that `FutureTask`.
-
-The tasks are cancelable, so that streams and other resource consuming handles can be released.
-
-{% top %}
diff --git a/docs/monitoring/application_profiling.md b/docs/monitoring/application_profiling.md
index 2a3feea9de16b287cfabda2a7c99327e83bb733f..3fe0adb372b630f026e966363068303aa5716e1a 100644
--- a/docs/monitoring/application_profiling.md
+++ b/docs/monitoring/application_profiling.md
@@ -1,5 +1,5 @@
---
-title: "Application Profiling"
+title: "Application Profiling & Debugging"
nav-parent_id: monitoring
nav-pos: 15
---
@@ -33,7 +33,7 @@ user in `env.java.opts`, `env.java.opts.jobmanager`, `env.java.opts.taskmanager`
use of the script variable `FLINK_LOG_PREFIX` and by enclosing the options in double quotes for late evaluation. Log files
using `FLINK_LOG_PREFIX` are rotated along with the default `.out` and `.log` files.
-# Profiling with Java Flight Recorder
+## Profiling with Java Flight Recorder
Java Flight Recorder is a profiling and event collection framework built into the Oracle JDK.
[Java Mission Control](http://www.oracle.com/technetwork/java/javaseproducts/mission-control/java-mission-control-1998576.html)
@@ -44,7 +44,7 @@ Flight Recorder. Example configuration:
env.java.opts: "-XX:+UnlockCommercialFeatures -XX:+UnlockDiagnosticVMOptions -XX:+FlightRecorder -XX:+DebugNonSafepoints -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=${FLINK_LOG_PREFIX}.jfr"
{% endhighlight %}
-# Profiling with JITWatch
+## Profiling with JITWatch
[JITWatch](https://github.com/AdoptOpenJDK/jitwatch/wiki) is a log analyser and visualizer for the Java HotSpot JIT
compiler used to inspect inlining decisions, hot methods, bytecode, and assembly. Example configuration:
@@ -53,4 +53,32 @@ compiler used to inspect inlining decisions, hot methods, bytecode, and assembly
env.java.opts: "-XX:+UnlockDiagnosticVMOptions -XX:+TraceClassLoading -XX:+LogCompilation -XX:LogFile=${FLINK_LOG_PREFIX}.jit -XX:+PrintAssembly"
{% endhighlight %}
+## Analyzing Out of Memory Problems
+
+If you encounter `OutOfMemoryExceptions` with your Flink application, then it is a good idea to enable heap dumps on out of memory errors.
+
+{% highlight yaml %}
+env.java.opts: "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${FLINK_LOG_PREFIX}.hprof"
+{% endhighlight %}
+
+The heap dump will allow you to analyze potential memory leaks in your user code.
+If the memory leak should be caused by Flink, then please reach out to the [dev mailing list](mailto:dev@flink.apache.org).
+
+## Analyzing Memory & Garbage Collection Behaviour
+
+Memory usage and garbage collection can have a profound impact on your application.
+The effects can range from slight performance degradation to a complete cluster failure if the GC pauses are too long.
+If you want to better understand the memory and GC behaviour of your application, then you can enable memory logging on the `TaskManagers`.
+
+{% highlight yaml %}
+taskmanager.debug.memory.log: true
+taskmanager.debug.memory.log-interval: 10000 // 10s interval
+{% endhighlight %}
+
+If you are interested in more detailed GC statistics, then you can activate the JVM's GC logging via:
+
+{% highlight yaml %}
+env.java.opts: "-Xloggc:${FLINK_LOG_PREFIX}.gc.log -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+PrintPromotionFailure -XX:+PrintGCCause"
+{% endhighlight %}
+
{% top %}
diff --git a/docs/monitoring/application_profiling.zh.md b/docs/monitoring/application_profiling.zh.md
index c766164d2db70d0dc5d5f1b5aea3fe9cae3aaa84..6061f214136055b35efff0e456fa4937283afa5b 100644
--- a/docs/monitoring/application_profiling.zh.md
+++ b/docs/monitoring/application_profiling.zh.md
@@ -33,7 +33,7 @@ user in `env.java.opts`, `env.java.opts.jobmanager`, `env.java.opts.taskmanager`
use of the script variable `FLINK_LOG_PREFIX` and by enclosing the options in double quotes for late evaluation. Log files
using `FLINK_LOG_PREFIX` are rotated along with the default `.out` and `.log` files.
-# Profiling with Java Flight Recorder
+## Profiling with Java Flight Recorder
Java Flight Recorder is a profiling and event collection framework built into the Oracle JDK.
[Java Mission Control](http://www.oracle.com/technetwork/java/javaseproducts/mission-control/java-mission-control-1998576.html)
@@ -44,7 +44,7 @@ Flight Recorder. Example configuration:
env.java.opts: "-XX:+UnlockCommercialFeatures -XX:+UnlockDiagnosticVMOptions -XX:+FlightRecorder -XX:+DebugNonSafepoints -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=${FLINK_LOG_PREFIX}.jfr"
{% endhighlight %}
-# Profiling with JITWatch
+## Profiling with JITWatch
[JITWatch](https://github.com/AdoptOpenJDK/jitwatch/wiki) is a log analyser and visualizer for the Java HotSpot JIT
compiler used to inspect inlining decisions, hot methods, bytecode, and assembly. Example configuration:
@@ -53,4 +53,32 @@ compiler used to inspect inlining decisions, hot methods, bytecode, and assembly
env.java.opts: "-XX:+UnlockDiagnosticVMOptions -XX:+TraceClassLoading -XX:+LogCompilation -XX:LogFile=${FLINK_LOG_PREFIX}.jit -XX:+PrintAssembly"
{% endhighlight %}
+## Analyzing Out of Memory Problems
+
+If you encounter `OutOfMemoryExceptions` with your Flink application, then it is a good idea to enable heap dumps on out of memory errors.
+
+{% highlight yaml %}
+env.java.opts: "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${FLINK_LOG_PREFIX}.hprof"
+{% endhighlight %}
+
+The heap dump will allow you to analyze potential memory leaks in your user code.
+If the memory leak should be caused by Flink, then please reach out to the [dev mailing list](mailto:dev@flink.apache.org).
+
+## Analyzing Memory & Garbage Collection Behaviour
+
+Memory usage and garbage collection can have a profound impact on your application.
+The effects can range from slight performance degradation to a complete cluster failure if the GC pauses are too long.
+If you want to better understand the memory and GC behaviour of your application, then you can enable memory logging on the `TaskManagers`.
+
+{% highlight yaml %}
+taskmanager.debug.memory.log: true
+taskmanager.debug.memory.log-interval: 10000 // 10s interval
+{% endhighlight %}
+
+If you are interested in more detailed GC statistics, then you can activate the JVM's GC logging via:
+
+{% highlight yaml %}
+env.java.opts: "-Xloggc:${FLINK_LOG_PREFIX}.gc.log -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+PrintPromotionFailure -XX:+PrintGCCause"
+{% endhighlight %}
+
{% top %}
diff --git a/docs/monitoring/back_pressure.md b/docs/monitoring/back_pressure.md
index 0cc4644c3e51b2b300c37fcadc2d1b08a9161817..c6bec13e4146d9532853744a4fc0d94430206784 100644
--- a/docs/monitoring/back_pressure.md
+++ b/docs/monitoring/back_pressure.md
@@ -34,30 +34,30 @@ If you see a **back pressure warning** (e.g. `High`) for a task, this means that
Take a simple `Source -> Sink` job as an example. If you see a warning for `Source`, this means that `Sink` is consuming data slower than `Source` is producing. `Sink` is back pressuring the upstream operator `Source`.
-## Sampling Threads
+## Sampling Back Pressure
-Back pressure monitoring works by repeatedly taking stack trace samples of your running tasks. The JobManager triggers repeated calls to `Thread.getStackTrace()` for the tasks of your job.
+Back pressure monitoring works by repeatedly taking back pressure samples of your running tasks. The JobManager triggers repeated calls to `Task.isBackPressured()` for the tasks of your job.
-
+
-If the samples show that a task Thread is stuck in a certain internal method call (requesting buffers from the network stack), this indicates that there is back pressure for the task.
+Internally, back pressure is judged based on the availability of output buffers. If there is no available buffer (at least one) for output, then it indicates that there is back pressure for the task.
-By default, the job manager triggers 100 stack traces every 50ms for each task in order to determine back pressure. The ratio you see in the web interface tells you how many of these stack traces were stuck in the internal method call, e.g. `0.01` indicates that only 1 in 100 was stuck in that method.
+By default, the job manager triggers 100 samples every 50ms for each task in order to determine back pressure. The ratio you see in the web interface tells you how many of these samples were indicating back pressure, e.g. `0.01` indicates that only 1 in 100 was back pressured.
- **OK**: 0 <= Ratio <= 0.10
- **LOW**: 0.10 < Ratio <= 0.5
- **HIGH**: 0.5 < Ratio <= 1
-In order to not overload the task managers with stack trace samples, the web interface refreshes samples only after 60 seconds.
+In order to not overload the task managers with back pressure samples, the web interface refreshes samples only after 60 seconds.
## Configuration
You can configure the number of samples for the job manager with the following configuration keys:
- `web.backpressure.refresh-interval`: Time after which available stats are deprecated and need to be refreshed (DEFAULT: 60000, 1 min).
-- `web.backpressure.num-samples`: Number of stack trace samples to take to determine back pressure (DEFAULT: 100).
-- `web.backpressure.delay-between-samples`: Delay between stack trace samples to determine back pressure (DEFAULT: 50, 50 ms).
+- `web.backpressure.num-samples`: Number of samples to take to determine back pressure (DEFAULT: 100).
+- `web.backpressure.delay-between-samples`: Delay between samples to determine back pressure (DEFAULT: 50, 50 ms).
## Example
@@ -66,7 +66,7 @@ You can find the *Back Pressure* tab next to the job overview.
### Sampling In Progress
-This means that the JobManager triggered a stack trace sample of the running tasks. With the default configuration, this takes about 5 seconds to complete.
+This means that the JobManager triggered a back pressure sample of the running tasks. With the default configuration, this takes about 5 seconds to complete.
Note that clicking the row, you trigger the sample for all subtasks of this operator.
diff --git a/docs/monitoring/back_pressure.zh.md b/docs/monitoring/back_pressure.zh.md
index 127f7ffd1af56811979b7048eaa890c68c568d50..4dd171d3644e40f4e13f7d362e9b7998c9597127 100644
--- a/docs/monitoring/back_pressure.zh.md
+++ b/docs/monitoring/back_pressure.zh.md
@@ -36,31 +36,31 @@ Flink Web 界面提供了一个选项卡来监控正在运行 Job 的反压行
`Sink` 正在向上游的 `Source` 算子产生反压。
-## 采样线程
+## 反压采样
-通过不断对每个 Task 的 stack trace 采样来进行反压监控。JobManager 会触发对 Task `Thread.getStackTrace()` 的重复调用。
+通过不断对每个 Task 的反压状态采样来进行反压监控。JobManager 会触发对 Task `Task.isBackPressured()` 的重复调用。
-
+
-如果样本显示 Task 线程卡在某个内部方法调用中（例如：从网络堆栈请求缓冲区），则表示这个 Task 存在反压。
+Task 是否反压是基于输出 Buffer 的可用性判断的，如果一个用于数据输出的 Buffer 都没有了，则表明 Task 被反压了。
-默认情况下，JobManager 会触发 100 次 stack trace 采样，每次间隔 50ms 来确定反压。
-你在 Web 界面看到的比率表示在内部方法调用中有多少 stack trace 被卡住，例如: `0.01` 表示在该方法中 100 个只有 1 个被卡住了。
+默认情况下，JobManager 会触发 100 次采样，每次间隔 50ms 来确定反压。
+你在 Web 界面看到的比率表示在获得的样本中有多少表明 Task 正在被反压，例如: `0.01` 表示 100 个样本中只有 1 个反压了。
- **OK**: 0 <= 比例 <= 0.10
- **LOW**: 0.10 < 比例 <= 0.5
- **HIGH**: 0.5 < 比例 <= 1
-为了不因为 stack trace 导致 TaskManager 产生负载，Web 界面仅在 60 秒后重新采样。
+为了不因为采样导致 TaskManager 负载过重，Web 界面仅在每 60 秒后重新采样。
## 配置参数
你可以使用以下键来配置 JobManager 的样本数：
-- `web.backpressure.refresh-interval`: 有效统计被废弃并重新进行采样的时间 (默认: 60000, 1 min)。
-- `web.backpressure.num-samples`: 用于确定反压的 stack trace 样本数 (默认: 100)。
-- `web.backpressure.delay-between-samples`: 用于确定反压的 stack trace 采样间隔时间 (默认: 50, 50 ms)。
+- `web.backpressure.refresh-interval`: 有效的反压结果被废弃并重新进行采样的时间 (默认: 60000, 1 min)。
+- `web.backpressure.num-samples`: 用于确定反压采样的样本数 (默认: 100)。
+- `web.backpressure.delay-between-samples`: 用于确定反压采样的间隔时间 (默认: 50, 50 ms)。
## 示例
@@ -69,7 +69,7 @@ Flink Web 界面提供了一个选项卡来监控正在运行 Job 的反压行
### 采样进行中
-这意味着 JobManager 对正在运行的 Task 触发了 stack trace 采样。默认情况下，大约需要 5 秒完成采样。
+这意味着 JobManager 对正在运行的 Task 触发了反压采样。默认情况下，大约需要 5 秒完成采样。
注意，点击该行，可触发该算子所有 SubTask 的采样。
diff --git a/docs/monitoring/logging.md b/docs/monitoring/logging.md
index 737ddb9de3031c228c91436a4592e333929df1b5..842a0afe3e1992164a3f314982e7bface63d6dc9 100644
--- a/docs/monitoring/logging.md
+++ b/docs/monitoring/logging.md
@@ -23,14 +23,14 @@ specific language governing permissions and limitations
under the License.
-->
-The logging in Flink is implemented using the slf4j logging interface. As underlying logging framework, log4j is used. We also provide logback configuration files and pass them to the JVM's as properties. Users willing to use logback instead of log4j can just exclude log4j (or delete it from the lib/ folder).
+The logging in Flink is implemented using the slf4j logging interface. As underlying logging framework, log4j2 is used. We also provide logback configuration files and pass them to the JVM's as properties. Users willing to use logback instead of log4j2 can just exclude log4j2 (or delete it from the lib/ folder).
* This will be replaced by the TOC
{:toc}
-## Configuring Log4j
+## Configuring Log4j2
-Log4j is controlled using property files. In Flink's case, the file is usually called `log4j.properties`. We pass the filename and location of this file using the `-Dlog4j.configuration=` parameter to the JVM.
+Log4j2 is controlled using property files. In Flink's case, the file is usually called `log4j.properties`. We pass the filename and location of this file using the `-Dlog4j.configurationFile=` parameter to the JVM.
Flink ships with the following default properties files:
@@ -38,6 +38,25 @@ Flink ships with the following default properties files:
- `log4j-yarn-session.properties`: Used by the Flink command line client when starting a YARN session (`yarn-session.sh`)
- `log4j.properties`: JobManager/Taskmanager logs (both standalone and YARN)
+### Compatibility with Log4j1
+
+Flink ships with the [Log4j API bridge](https://logging.apache.org/log4j/log4j-2.2/log4j-1.2-api/index.html), allowing existing applications that work against Log4j1 classes to continue working.
+
+If you have custom Log4j1 properties files or code that relies on Log4j1, please check out the official Log4j [compatibility](https://logging.apache.org/log4j/2.x/manual/compatibility.html) and [migration](https://logging.apache.org/log4j/2.x/manual/migration.html) guides.
+
+## Configuring Log4j1
+
+To use Flink with Log4j1 you must ensure that:
+- `org.apache.logging.log4j:log4j-core`, `org.apache.logging.log4j:log4j-slf4j-impl` and `org.apache.logging.log4j:log4j-1.2-api` are not on the classpath,
+- `log4j:log4j`, `org.slf4j:slf4j-log4j12`, `org.apache.logging.log4j:log4j-to-slf4j` and `org.apache.logging.log4j:log4j-api` are on the classpath.
+
+In the IDE this means you have to replace such dependencies defined in your pom, and possibly add exclusions on dependencies that transitively depend on them.
+
+For Flink distributions this means you have to
+- remove the `log4j-core`, `log4j-slf4j-impl` and `log4j-1.2-api` jars from the `lib` directory,
+- add the `log4j`, `slf4j-log4j12` and `log4j-to-slf4j` jars to the `lib` directory,
+- replace all log4j properties files in the `conf` directory with Log4j1-compliant versions.
+
## Configuring logback
For users and developers alike it is important to control the logging framework.
diff --git a/docs/monitoring/logging.zh.md b/docs/monitoring/logging.zh.md
index 3b6d7e681e77734b387b8ff8d7a173e480151b4a..ec0b85bfbeed79c6c6f897736291231f2a8b6419 100644
--- a/docs/monitoring/logging.zh.md
+++ b/docs/monitoring/logging.zh.md
@@ -23,14 +23,14 @@ specific language governing permissions and limitations
under the License.
-->
-The logging in Flink is implemented using the slf4j logging interface. As underlying logging framework, log4j is used. We also provide logback configuration files and pass them to the JVM's as properties. Users willing to use logback instead of log4j can just exclude log4j (or delete it from the lib/ folder).
+The logging in Flink is implemented using the slf4j logging interface. As underlying logging framework, log4j2 is used. We also provide logback configuration files and pass them to the JVM's as properties. Users willing to use logback instead of log4j2 can just exclude log4j2 (or delete it from the lib/ folder).
* This will be replaced by the TOC
{:toc}
-## Configuring Log4j
+## Configuring Log4j2
-Log4j is controlled using property files. In Flink's case, the file is usually called `log4j.properties`. We pass the filename and location of this file using the `-Dlog4j.configuration=` parameter to the JVM.
+Log4j2 is controlled using property files. In Flink's case, the file is usually called `log4j.properties`. We pass the filename and location of this file using the `-Dlog4j.configurationFile=` parameter to the JVM.
Flink ships with the following default properties files:
@@ -38,6 +38,25 @@ Flink ships with the following default properties files:
- `log4j-yarn-session.properties`: Used by the Flink command line client when starting a YARN session (`yarn-session.sh`)
- `log4j.properties`: JobManager/Taskmanager logs (both standalone and YARN)
+### Compatibility with Log4j1
+
+Flink ships with the [Log4j API bridge](https://logging.apache.org/log4j/log4j-2.2/log4j-1.2-api/index.html), allowing existing applications that work against Log4j1 classes to continue working.
+
+If you have custom Log4j1 properties files or code that relies on Log4j1, please check out the official Log4j [compatibility](https://logging.apache.org/log4j/2.x/manual/compatibility.html) and [migration](https://logging.apache.org/log4j/2.x/manual/migration.html) guides.
+
+## Configuring Log4j1
+
+To use Flink with Log4j1 you must ensure that:
+- `org.apache.logging.log4j:log4j-core`, `org.apache.logging.log4j:log4j-slf4j-impl` and `org.apache.logging.log4j:log4j-1.2-api` are not on the classpath,
+- `log4j:log4j`, `org.slf4j:slf4j-log4j12`, `org.apache.logging.log4j:log4j-to-slf4j` and `org.apache.logging.log4j:log4j-api` are on the classpath.
+
+In the IDE this means you have to replace such dependencies defined in your pom, and possibly add exclusions on dependencies that transitively depend on them.
+
+For Flink distributions this means you have to
+- remove the `log4j-core`, `log4j-slf4j-impl` and `log4j-1.2-api` jars from the `lib` directory,
+- add the `log4j`, `slf4j-log4j12` and `log4j-to-slf4j` jars to the `lib` directory,
+- replace all log4j properties files in the `conf` directory with Log4j1-compliant versions.
+
## Configuring logback
For users and developers alike it is important to control the logging framework.
diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md
index 696b508ae4301579dfb591bdb7a3a9405e7b112a..53e0bedac6c8cada8c73fb52ad81b43e962ce7a4 100644
--- a/docs/monitoring/metrics.md
+++ b/docs/monitoring/metrics.md
@@ -433,7 +433,7 @@ You can configure which delimiter to use for the identifier (default: `.`) by se
### User Scope
-You can define a user scope by calling `MetricGroup#addGroup(String name)`, `MetricGroup#addGroup(int name)` or `Metric#addGroup(String key, String value)`.
+You can define a user scope by calling `MetricGroup#addGroup(String name)`, `MetricGroup#addGroup(int name)` or `MetricGroup#addGroup(String key, String value)`.
These methods affect what `MetricGroup#getMetricIdentifier` and `MetricGroup#getScopeComponents` return.
@@ -563,6 +563,7 @@ reporters will be instantiated on each job and task manager when they are starte
- `metrics.reporter..factory.class`: The reporter factory class to use for the reporter named ``.
- `metrics.reporter..interval`: The reporter interval to use for the reporter named ``.
- `metrics.reporter..scope.delimiter`: The delimiter to use for the identifier (default value use `metrics.scope.delimiter`) for the reporter named ``.
+- `metrics.reporter..scope.variables.excludes`: (optional) A semi-colon (;) separate list of variables that should be ignored by tag-based reporters (e.g., Prometheus, InfluxDB).
- `metrics.reporters`: (optional) A comma-separated include list of reporter names. By default all configured reporters will be used.
All reporters must at least have either the `class` or `factory.class` property. Which property may/should be used depends on the reporter implementation. See the individual reporter configuration sections for more information.
@@ -576,6 +577,7 @@ metrics.reporters: my_jmx_reporter,my_other_reporter
metrics.reporter.my_jmx_reporter.factory.class: org.apache.flink.metrics.jmx.JMXReporterFactory
metrics.reporter.my_jmx_reporter.port: 9020-9040
+metrics.reporter.my_jmx_reporter.scope.variables.excludes:job_id;task_attempt_num
metrics.reporter.my_other_reporter.class: org.apache.flink.metrics.graphite.GraphiteReporter
metrics.reporter.my_other_reporter.host: 192.168.1.1
@@ -653,14 +655,7 @@ of your Flink distribution.
Parameters:
-- `host` - the InfluxDB server host
-- `port` - (optional) the InfluxDB server port, defaults to `8086`
-- `db` - the InfluxDB database to store metrics
-- `username` - (optional) InfluxDB username used for authentication
-- `password` - (optional) InfluxDB username's password used for authentication
-- `retentionPolicy` - (optional) InfluxDB retention policy, defaults to retention policy defined on the server for the db
-- `connectTimeout` - (optional) the InfluxDB client connect timeout in milliseconds, default is 10000 ms
-- `writeTimeout` - (optional) the InfluxDB client write timeout in milliseconds, default is 10000 ms
+{% include generated/influxdb_reporter_configuration.html %}
Example configuration:
@@ -673,6 +668,7 @@ metrics.reporter.influxdb.db: flink
metrics.reporter.influxdb.username: flink-metrics
metrics.reporter.influxdb.password: qwerty
metrics.reporter.influxdb.retentionPolicy: one_hour
+metrics.reporter.influxdb.consistency: ANY
metrics.reporter.influxdb.connectTimeout: 60000
metrics.reporter.influxdb.writeTimeout: 60000
@@ -1033,8 +1029,8 @@ Thus, in order to infer the metric identifier:
Gauge
-
Task
-
buffers
+
Task
+
buffers
inputQueueLength
The number of queued input buffers. (ignores LocalInputChannels which are using blocking subpartitions)
Gauge
@@ -1051,12 +1047,12 @@ Thus, in order to infer the metric identifier:
inputFloatingBuffersUsage
-
An estimate of the floating input buffers usage, dedicated for credit-based mode. (ignores LocalInputChannels)
+
An estimate of the floating input buffers usage. (ignores LocalInputChannels)
Gauge
inputExclusiveBuffersUsage
-
An estimate of the exclusive input buffers usage, dedicated for credit-based mode. (ignores LocalInputChannels)
+
An estimate of the exclusive input buffers usage. (ignores LocalInputChannels)
Gauge
@@ -1253,7 +1249,7 @@ Metrics related to data exchange between task executors using netty network comm
-
Job (only available on JobManager)
+
Job (only available on JobManager)
restartingTime
The time it took to restart the job, or how long the current restart has been in progress (in milliseconds).
Gauge
@@ -1276,7 +1272,12 @@ Metrics related to data exchange between task executors using netty network comm
fullRestarts
-
The total number of full restarts since this job was submitted.
+
Attention: deprecated, use numRestarts.
+
Gauge
+
+
+
numRestarts
+
The total number of restarts since this job was submitted, including full restarts and fine-grained restarts.
Gauge
@@ -1340,11 +1341,16 @@ Metrics related to data exchange between task executors using netty network comm
Gauge
-
Task
+
Task
checkpointAlignmentTime
The time in nanoseconds that the last barrier alignment took to complete, or how long the current alignment has taken so far (in nanoseconds).
Gauge
+
+
checkpointStartDelayNanos
+
The time in nanoseconds that elapsed between the creation of the last checkpoint and the time when the checkpointing process has started by this Task. This delay shows how long it takes for the first checkpoint barrier to reach the task. A high value indicates back-pressure. If only a specific task has a long start delay, the most likely reason is data skew.
+
Gauge
+
@@ -1365,11 +1371,11 @@ Certain RocksDB native metrics are available but disabled by default, you can fi
The latency distributions from a given source (subtask) to an operator subtask (in milliseconds), depending on the [latency granularity]({{ site.baseurl }}/ops/config.html#metrics-latency-granularity).
+
The latency distributions from a given source (subtask) to an operator subtask (in milliseconds), depending on the latency granularity.
@@ -1429,6 +1435,11 @@ Certain RocksDB native metrics are available but disabled by default, you can fi
The number of network buffers this task emits per second.
Meter
+
+
isBackPressured
+
Whether the task is back-pressured.
+
Gauge
+
Task/Operator
numRecordsIn
diff --git a/docs/monitoring/metrics.zh.md b/docs/monitoring/metrics.zh.md
index 389c65cabf6c661f171e59b4d1033b16787d4e55..58763c58ebf330f4df5021c051d94b2f8af3da7c 100644
--- a/docs/monitoring/metrics.zh.md
+++ b/docs/monitoring/metrics.zh.md
@@ -433,7 +433,7 @@ You can configure which delimiter to use for the identifier (default: `.`) by se
### User Scope
-You can define a user scope by calling `MetricGroup#addGroup(String name)`, `MetricGroup#addGroup(int name)` or `Metric#addGroup(String key, String value)`.
+You can define a user scope by calling `MetricGroup#addGroup(String name)`, `MetricGroup#addGroup(int name)` or `MetricGroup#addGroup(String key, String value)`.
These methods affect what `MetricGroup#getMetricIdentifier` and `MetricGroup#getScopeComponents` return.
@@ -563,6 +563,7 @@ reporters will be instantiated on each job and task manager when they are starte
- `metrics.reporter..factory.class`: The reporter factory class to use for the reporter named ``.
- `metrics.reporter..interval`: The reporter interval to use for the reporter named ``.
- `metrics.reporter..scope.delimiter`: The delimiter to use for the identifier (default value use `metrics.scope.delimiter`) for the reporter named ``.
+- `metrics.reporter..scope.variables.excludes`: (optional) A semi-colon (;) separate list of variables that should be ignored by tag-based reporters (e.g., Prometheus, InfluxDB).
- `metrics.reporters`: (optional) A comma-separated include list of reporter names. By default all configured reporters will be used.
All reporters must at least have either the `class` or `factory.class` property. Which property may/should be used depends on the reporter implementation. See the individual reporter configuration sections for more information.
@@ -576,6 +577,7 @@ metrics.reporters: my_jmx_reporter,my_other_reporter
metrics.reporter.my_jmx_reporter.factory.class: org.apache.flink.metrics.jmx.JMXReporterFactory
metrics.reporter.my_jmx_reporter.port: 9020-9040
+metrics.reporter.my_jmx_reporter.scope.variables.excludes:job_id;task_attempt_num
metrics.reporter.my_other_reporter.class: org.apache.flink.metrics.graphite.GraphiteReporter
metrics.reporter.my_other_reporter.host: 192.168.1.1
@@ -653,14 +655,7 @@ of your Flink distribution.
Parameters:
-- `host` - the InfluxDB server host
-- `port` - (optional) the InfluxDB server port, defaults to `8086`
-- `db` - the InfluxDB database to store metrics
-- `username` - (optional) InfluxDB username used for authentication
-- `password` - (optional) InfluxDB username's password used for authentication
-- `retentionPolicy` - (optional) InfluxDB retention policy, defaults to retention policy defined on the server for the db
-- `connectTimeout` - (optional) the InfluxDB client connect timeout in milliseconds, default is 10000 ms
-- `writeTimeout` - (optional) the InfluxDB client write timeout in milliseconds, default is 10000 ms
+{% include generated/influxdb_reporter_configuration.html %}
Example configuration:
@@ -673,6 +668,7 @@ metrics.reporter.influxdb.db: flink
metrics.reporter.influxdb.username: flink-metrics
metrics.reporter.influxdb.password: qwerty
metrics.reporter.influxdb.retentionPolicy: one_hour
+metrics.reporter.influxdb.consistency: ANY
metrics.reporter.influxdb.connectTimeout: 60000
metrics.reporter.influxdb.writeTimeout: 60000
@@ -1033,8 +1029,8 @@ Thus, in order to infer the metric identifier:
Gauge
-
Task
-
buffers
+
Task
+
buffers
inputQueueLength
The number of queued input buffers. (ignores LocalInputChannels which are using blocking subpartitions)
Gauge
@@ -1051,12 +1047,12 @@ Thus, in order to infer the metric identifier:
inputFloatingBuffersUsage
-
An estimate of the floating input buffers usage, dedicated for credit-based mode. (ignores LocalInputChannels)
+
An estimate of the floating input buffers usage. (ignores LocalInputChannels)
Gauge
inputExclusiveBuffersUsage
-
An estimate of the exclusive input buffers usage, dedicated for credit-based mode. (ignores LocalInputChannels)
+
An estimate of the exclusive input buffers usage. (ignores LocalInputChannels)
Gauge
@@ -1253,7 +1249,7 @@ Metrics related to data exchange between task executors using netty network comm
-
Job (only available on JobManager)
+
Job (only available on JobManager)
restartingTime
The time it took to restart the job, or how long the current restart has been in progress (in milliseconds).
Gauge
@@ -1276,7 +1272,12 @@ Metrics related to data exchange between task executors using netty network comm
fullRestarts
-
The total number of full restarts since this job was submitted.
+
Attention: deprecated, use numRestarts.
+
Gauge
+
+
+
numRestarts
+
The total number of restarts since this job was submitted, including full restarts and fine-grained restarts.
Gauge
@@ -1340,11 +1341,16 @@ Metrics related to data exchange between task executors using netty network comm
Gauge
-
Task
+
Task
checkpointAlignmentTime
The time in nanoseconds that the last barrier alignment took to complete, or how long the current alignment has taken so far (in nanoseconds).
Gauge
+
+
checkpointStartDelayNanos
+
The time in nanoseconds that elapsed between the creation of the last checkpoint and the time when the checkpointing process has started by this Task. This delay shows how long it takes for the first checkpoint barrier to reach the task. A high value indicates back-pressure. If only a specific task has a long start delay, the most likely reason is data skew.
+
Gauge
+
@@ -1365,11 +1371,11 @@ Certain RocksDB native metrics are available but disabled by default, you can fi
The latency distributions from a given source (subtask) to an operator subtask (in milliseconds), depending on the [latency granularity]({{ site.baseurl }}/ops/config.html#metrics-latency-granularity).
+
The latency distributions from a given source (subtask) to an operator subtask (in milliseconds), depending on the latency granularity.
@@ -1429,6 +1435,11 @@ Certain RocksDB native metrics are available but disabled by default, you can fi
The number of network buffers this task emits per second.
Meter
+
+
isBackPressured
+
Whether the task is back-pressured.
+
Gauge
+
Task/Operator
numRecordsIn
diff --git a/docs/ops/cli.md b/docs/ops/cli.md
index cb63e9cd1cd9e4ea3f4deb1cf8c44e07f5f5004d..04e97ffa2a91a35eb51b98ae105da66d9b417434 100644
--- a/docs/ops/cli.md
+++ b/docs/ops/cli.md
@@ -2,7 +2,7 @@
title: "Command-Line Interface"
nav-title: CLI
nav-parent_id: ops
-nav-pos: 6
+nav-pos: 7
---
-**For single-node setups Flink is ready to go out of the box and you don't need to change the default configuration to get started.**
+All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+
+The configuration is parsed and evaluated when the Flink processes are started. Changes to the configuration file require restarting the relevant processes.
The out of the box configuration will use your default Java installation. You can manually set the environment variable `JAVA_HOME` or the configuration key `env.java.home` in `conf/flink-conf.yaml` if you want to manually override the Java runtime to use.
-This page lists the most common options that are typically needed to set up a well performing (distributed) installation. In addition a full list of all available configuration parameters is listed here.
+* This will be replaced by the TOC
+{:toc}
-All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+# Basic Setup
-The system and run scripts parse the config at startup time. Changes to the configuration file require restarting the Flink JobManager and TaskManagers.
+The default configuration supports starting a single-node Flink session cluster without any changes.
+The options in this section are the ones most commonly needed for a basic distributed Flink setup.
-The configuration files for the TaskManagers can be different, Flink does not assume uniform machines in the cluster.
+**Hostnames / Ports**
-* This will be replaced by the TOC
-{:toc}
+These options are only necessary for *standalone* application- or session deployments ([simple standalone]({{site.baseurl}}/ops/deployment/cluster_setup.html) or [Kubernetes]({{site.baseurl}}/ops/deployment/kubernetes.html)).
-## Common Options
+If you use Flink with [Yarn]({{site.baseurl}}/ops/deployment/yarn_setup.html), [Mesos]({{site.baseurl}}/ops/deployment/mesos.html), or the [*active* Kubernetes integration]({{site.baseurl}}/ops/deployment/native_kubernetes.html), the hostnames and ports are automatically discovered.
-{% include generated/common_section.html %}
+ - `rest.address`, `rest.port`: These are used by the client to connect to Flink. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes) service in front of the Flink Master's REST interface.
-## Full Reference
+ - The `jobmanager.rpc.address` (defaults to *"localhost"*) and `jobmanager.rpc.port` (defaults to *6123*) config entries are used by the TaskManager to connect to the JobManager/ResourceManager. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes internal) service for the Flink master (JobManager). This option is ignored on [setups with high-availability]({{site.baseurl}}/ops/jobmanager_high_availability.html) where the leader election mechanism is used to discover this automatically.
-### HDFS
+**Memory Sizes**
-
- Note: These keys are deprecated and it is recommended to configure the Hadoop path with the environment variable HADOOP_CONF_DIR instead.
-
+The default memory sizes support simple streaming/batch applications, but are too low to yield good performance for more complex applications.
+
+ - `jobmanager.heap.size`: Sets the size of the *Flink Master* (JobManager / ResourceManager / Dispatcher) JVM heap.
+ - `taskmanager.memory.process.size`: Total size of the TaskManager process, including everything. Flink will subtract some memory for the JVM's own memory requirements (metaspace and others), and divide and configure the rest automatically between its components (network, managed memory, JVM Heap, etc.).
+
+These value are configured as memory sizes, for example *1536m* or *2g*.
+
+**Parallelism**
+
+ - `taskmanager.numberOfTaskSlots`: The number of slots that a TaskManager offers *(default: 1)*. Each slot can take one task or pipeline.
+ Having multiple slots in a TaskManager can help amortize certain constant overheads (of the JVM, application libraries, or network connections) across parallel tasks or pipelines. See the [Task Slots and Resources]({{site.baseurl}}/concepts/runtime.html#task-slots-and-resources) concepts section for details.
-See also [how to configure Hadoop]({{ site.baseurl }}/ops/deployment/hadoop.html#configure-hadoop).
+ Running more smaller TaskManagers with one slot each is a good starting point and leads to the best isolation between tasks. Dedicating the same resources to fewer larger TaskManagers with more slots can help to increase resource utilization, at the cost of weaker isolation between the tasks (more tasks share the same JVM).
-These parameters configure the default HDFS used by Flink. Setups that do not specify an HDFS configuration have to specify the full path to HDFS files (`hdfs://address:port/path/to/files`) Files will also be written with default HDFS parameters (block size, replication factor).
+ - `parallelism.default`: The default parallelism used when no parallelism is specified anywhere *(default: 1)*.
-- `fs.hdfs.hadoopconf`: The absolute path to the Hadoop File System's (HDFS) configuration **directory** (OPTIONAL VALUE). Specifying this value allows programs to reference HDFS files using short URIs (`hdfs:///path/to/files`, without including the address and port of the NameNode in the file URI). Without this option, HDFS files can be accessed, but require fully qualified URIs like `hdfs://address:port/path/to/files`. This option also causes file writers to pick up the HDFS's default values for block sizes and replication factors. Flink will look for the "core-site.xml" and "hdfs-site.xml" files in the specified directory.
+**Checkpointing**
-- `fs.hdfs.hdfsdefault`: The absolute path of Hadoop's own configuration file "hdfs-default.xml" (DEFAULT: null).
+You can configure checkpointing directly in code within your Flink job or application. Putting these values here in the configuration defines them as defaults in case the application does not configure anything.
-- `fs.hdfs.hdfssite`: The absolute path of Hadoop's own configuration file "hdfs-site.xml" (DEFAULT: null).
+ - `state.backend`: The state backend to use. This defines the data structure mechanism for taking snapshots. Common values are `filesystem` or `rocksdb`.
+ - `state.checkpoints.dir`: The directory to write checkpoints to. This takes a path URI like *s3://mybucket/flink-app/checkpoints* or *hdfs://namenode:port/flink/checkpoints*.
+ - `state.savepoints.dir`: The default directory for savepoints. Takes a path URI, similar to `state.checkpoints.dir`.
-### Core
+**Web UI**
-{% include generated/core_configuration.html %}
+ - `web.submit.enable`: Enables uploading and starting jobs through the Flink UI *(true by default)*. Please note that even when this is disabled, session clusters still accept jobs through REST requests (HTTP calls). This flag only guards the feature to upload jobs in the UI.
+ - `web.upload.dir`: The directory where to store uploaded jobs. Only used when `web.submit.enable` is true.
-### JobManager
+**Other**
-{% include generated/job_manager_configuration.html %}
+ - `io.tmp.dirs`: The directories where Flink puts local data, defaults to the system temp directory (`java.io.tmpdir` property). If a list of directories is configured, Flink will rotate files across the directories.
+
+ The data put in these directories include by default the files created by RocksDB, spilled intermediate results (batch algorithms), and cached jar files.
+
+ This data is NOT relied upon for persistence/recovery, but if this data gets deleted, it typically causes a heavyweight recovery operation. It is hence recommended to set this to a directory that is not automatically periodically purged.
+
+ Yarn, Mesos, and Kubernetes setups automatically configure this value to the local working directories by default.
-### Restart Strategies
+----
+----
-Configuration options to control Flink's restart behaviour in case of job failures.
+# Common Setup Options
+
+*Common options to configure your Flink application or cluster.*
+
+### Hosts and Ports
+
+Options to configure hostnames and ports for the different Flink components.
+
+The JobManager hostname and port are only relevant for standalone setups without high-availability.
+In that setup, the config values are used by the TaskManagers to find (and connect to) the JobManager.
+In all highly-available setups, the TaskManagers discover the JobManager via the High-Availability-Service (for example ZooKeeper).
+
+Setups using resource orchestration frameworks (K8s, Yarn, Mesos) typically use the framework's service discovery facilities.
+
+You do not need to configure any TaskManager hosts and ports, unless the setup requires the use of specific port ranges or specific network interfaces to bind to.
+
+{% include generated/common_host_port_section.html %}
+
+### Fault Tolerance
+
+These configuration options control Flink's restart behaviour in case of failures during the execution.
+By configuring these options in your `flink-conf.yaml`, you define the cluster's default restart strategy.
+
+The default restart strategy will only take effect if no job specific restart strategy has been configured via the `ExecutionConfig`.
{% include generated/restart_strategy_configuration.html %}
-#### Fixed Delay Restart Strategy
+**Fixed Delay Restart Strategy**
{% include generated/fixed_delay_restart_strategy_configuration.html %}
-#### Failure Rate Restart Strategy
+**Failure Rate Restart Strategy**
{% include generated/failure_rate_restart_strategy_configuration.html %}
-### TaskManager
+### Checkpoints and State Backends
-{% include generated/task_manager_configuration.html %}
+These options control the basic setup of state backends and checkpointing behavior.
-For *batch* jobs (or if `taskmanager.memoy.preallocate` is enabled) Flink allocates a fraction of 0.7 of the free memory (total memory configured via taskmanager.heap.size minus memory used for network buffers) for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents OutOfMemoryExceptions because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize the data to convert it into Java objects. All in all, managed memory improves the robustness and speed of the system.
+The options are only relevant for jobs/applications executing in a continuous streaming fashion.
+Jobs/applications executing in a batch fashion do not use state backends and checkpoints, but different internal data structures that are optimized for batch processing.
-The default fraction for managed memory can be adjusted using the taskmanager.memory.fraction parameter. An absolute value may be set using taskmanager.memory.size (overrides the fraction parameter). If desired, the managed memory may be allocated outside the JVM heap. This may improve performance in setups with large memory sizes.
+{% include generated/common_state_backends_section.html %}
-{% include generated/task_manager_memory_configuration.html %}
+### High Availability
-### Distributed Coordination
+High-availability here refers to the ability of the master (JobManager) process to recover from failures.
-{% include generated/cluster_configuration.html %}
+The JobManager ensures consistency during recovery across TaskManagers. For the JobManager itself to recover consistently, an external service must store a minimal amount of recovery metadata (like "ID of last committed checkpoint"), as well as help to elect and lock which JobManager is the leader (to avoid split-brain situations).
-### Distributed Coordination (via Akka)
+{% include generated/common_high_availability_section.html %}
-{% include generated/akka_configuration.html %}
+**Options for high-availability setups with ZooKeeper**
-### REST
+{% include generated/common_high_availability_zk_section.html %}
-{% include generated/rest_configuration.html %}
+### Memory Configuration
-### Blob Server
+These configuration values control the way that TaskManagers and JobManagers use memory.
-{% include generated/blob_server_configuration.html %}
+Flink tries to shield users as much as possible from the complexity of configuring the JVM for data-intensive processing.
+In most cases, users should only need to set the values `taskmanager.memory.process.size` or `taskmanager.memory.flink.size` (depending on how the setup), and possibly adjusting the ratio of JVM heap and Managed Memory via `taskmanager.memory.managed.fraction`. The other options below can be used for performane tuning and fixing memory related errors.
-### Heartbeat Manager
+For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory/mem_setup.html).
-{% include generated/heartbeat_manager_configuration.html %}
+{% include generated/common_memory_section.html %}
-### SSL Settings
+### Miscellaneous Options
-{% include generated/security_configuration.html %}
+{% include generated/common_miscellaneous_section.html %}
-### Netty Shuffle Environment
+----
+----
-{% include generated/netty_shuffle_environment_configuration.html %}
+# Security
-### Network Communication (via Netty)
+Options for configuring Flink's security and secure interaction with external systems.
-These parameters allow for advanced tuning. The default values are sufficient when running concurrent high-throughput jobs on a large cluster.
+### SSL
-{% include generated/network_netty_configuration.html %}
+Flink's network connections can be secured via SSL. Please refer to the [SSL Setup Docs]({{site.baseurl}}/ops/security-ssl.html) for detailed setup guide and background.
-### Web Frontend
+{% include generated/security_ssl_section.html %}
-{% include generated/web_configuration.html %}
-### File Systems
+### Auth with External Systems
-{% include generated/file_system_configuration.html %}
+**ZooKeeper Authentication / Authorization**
-### Compiler/Optimizer
+These options are necessary when connecting to a secured ZooKeeper quorum.
-{% include generated/optimizer_configuration.html %}
+{% include generated/security_auth_zk_section.html %}
-### Runtime Algorithms
+**Kerberos-based Authentication / Authorization**
-{% include generated/algorithm_configuration.html %}
+Please refer to the [Flink and Kerberos Docs]({{site.baseurl}}/ops/security-kerberos.html) for a setup guide and a list of external system to which Flink can authenticate itself via Kerberos.
-### Resource Manager
+{% include generated/security_auth_kerberos_section.html %}
-The configuration keys in this section are independent of the used resource management framework (YARN, Mesos, Standalone, ...)
+----
+----
-{% include generated/resource_manager_configuration.html %}
+# Resource Orchestration Frameworks
-### Shuffle Service
+This section contains options related to integrating Flink with resource orchestration frameworks, like Kubernetes, Yarn, Mesos, etc.
-{% include generated/shuffle_service_configuration.html %}
+Note that is not always necessary to integrate Flink with the resource orchestration framework.
+For example, you can easily deploy Flink applications on Kubernetes without Flink knowing that it runs on Kubernetes (and without specifying any of the Kubernetes config options here.) See [this setup guide]({{site.baseurl}}/ops/deployment/kubernetes.html) for an example.
+
+The options in this section are necessary for setups where Flink itself actively requests and releases resources from the orchestrators.
### YARN
{% include generated/yarn_config_configuration.html %}
+### Kubernetes
+
+{% include generated/kubernetes_config_configuration.html %}
+
### Mesos
{% include generated/mesos_configuration.html %}
-#### Mesos TaskManager
+**Mesos TaskManager**
{% include generated/mesos_task_manager_configuration.html %}
-### High Availability (HA)
+----
+----
-{% include generated/high_availability_configuration.html %}
+# State Backends
-#### ZooKeeper-based HA Mode
+Please refer to the [State Backend Documentation]({{site.baseurl}}/ops/state/state_backends.html) for background on State Backends.
-{% include generated/high_availability_zookeeper_configuration.html %}
+### RocksDB State Backend
-### ZooKeeper Security
+These are the options commonly needed to configure the RocksDB state backend. See the [Advanced RocksDB Backend Section](#advanced-rocksdb-state-backends-options) for options necessary for advanced low level configurations and trouble-shooting.
-{% include generated/zoo_keeper_configuration.html %}
+{% include generated/state_backend_rocksdb_section.html %}
-### Kerberos-based Security
+----
+----
-{% include generated/kerberos_configuration.html %}
+# Metrics
-### Environment
+Please refer to the [metrics system documentation]({{site.baseurl}}/monitoring/metrics.html) for background on Flink's metrics infrastructure.
-{% include generated/environment_configuration.html %}
+{% include generated/metric_configuration.html %}
-### Checkpointing
+### RocksDB Native Metrics
-{% include generated/checkpointing_configuration.html %}
+Flink can report metrics from RocksDB's native code, for applications using the RocksDB state backend.
+The metrics here are scoped to the operators and then further broken down by column family; values are reported as unsigned longs.
-### RocksDB State Backend
+
+ Note: Enabling RocksDB's native metrics may cause degraded performance and should be set carefully.
+
-{% include generated/rocks_db_configuration.html %}
+{% include generated/rocks_db_native_metric_configuration.html %}
-### RocksDB Configurable Options
-Specific RocksDB configurable options, provided by Flink, to create a corresponding `ConfigurableOptionsFactory`.
-And the created one would be used as default `OptionsFactory` in `RocksDBStateBackend`
-unless user define a `OptionsFactory` and set via `RocksDBStateBackend.setOptions(optionsFactory)`
+----
+----
-{% include generated/rocks_db_configurable_configuration.html %}
+# History Server
+
+The history server keeps the information of completed jobs (graphs, runtimes, statistics). To enable it, you have to enable "job archiving" in the JobManager (`jobmanager.archive.fs.dir`).
+
+See the [History Server Docs]({{site.baseurl}}/monitoring/historyserver.html) for details.
+
+{% include generated/history_server_configuration.html %}
+
+----
+----
+
+# Experimental
+
+*Options for experimental features in Flink.*
### Queryable State
+*Queryable State* is an experimental features that gives lets you access Flink's internal state like a key/value store.
+See the [Queryable State Docs]({{site.baseurl}}/dev/stream/state/queryable_state.html) for details.
+
{% include generated/queryable_state_configuration.html %}
-### Metrics
+----
+----
-{% include generated/metric_configuration.html %}
-
-### RocksDB Native Metrics
-Certain RocksDB native metrics may be forwarded to Flink's metrics reporter.
-All native metrics are scoped to operators and then further broken down by column family; values are reported as unsigned longs.
+# Debugging & Expert Tuning
- Note: Enabling native metrics may cause degraded performance and should be set carefully.
+ The options below here are meant for expert users and for fixing/debugging problems. Most setups should not need to configure these options.
-{% include generated/rocks_db_native_metric_configuration.html %}
+### Class Loading
-### History Server
+Flink dynamically loads the code for jobs submitted to a session cluster. In addition, Flink tries to hide many dependencies in the classpath from the application. This helps to reduce dependency conflicts between the application code and the dependencies in the classpath.
-You have to configure `jobmanager.archive.fs.dir` in order to archive terminated jobs and add it to the list of monitored directories via `historyserver.archive.fs.dir` if you want to display them via the HistoryServer's web frontend.
+Please refer to the [Debugging Classloading Docs]({{site.baseurl}}/monitoring/debugging_classloading.html) for details.
-- `jobmanager.archive.fs.dir`: Directory to upload information about terminated jobs to. You have to add this directory to the list of monitored directories of the history server via `historyserver.archive.fs.dir`.
+{% include generated/expert_class_loading_section.html %}
-{% include generated/history_server_configuration.html %}
+### Advanced State Backends Options
-## Legacy
+{% include generated/expert_state_backends_section.html %}
-- `mode`: Execution mode of Flink. Possible values are `legacy` and `new`. In order to start the legacy components, you have to specify `legacy` (DEFAULT: `new`).
+### Advanced RocksDB State Backends Options
-## Background
+Advanced options to tune RocksDB and RocksDB checkpoints.
-### Configuring the Network Buffers
+{% include generated/expert_rocksdb_section.html %}
-If you ever see the Exception `java.io.IOException: Insufficient number of network buffers`, you
-need to adapt the amount of memory used for network buffers in order for your program to run on your
-task managers.
+**RocksDB Configurable Options**
-Network buffers are a critical resource for the communication layers. They are used to buffer
-records before transmission over a network, and to buffer incoming data before dissecting it into
-records and handing them to the application. A sufficient number of network buffers is critical to
-achieve a good throughput.
+These options give fine-grained control over the behavior and resoures of ColumnFamilies.
+With the introduction of `state.backend.rocksdb.memory.managed` and `state.backend.rocksdb.memory.fixed-per-slot` (Apache Flink 1.10), it should be only necessary to use the options here for advanced performance tuning. These options here can also be specified in the application program via `RocksDBStateBackend.setOptions(PptionsFactory)`.
-
-Since Flink 1.3, you may follow the idiom "more is better" without any penalty on the latency (we
-prevent excessive buffering in each outgoing and incoming channel, i.e. *buffer bloat*, by limiting
-the actual number of buffers used by each channel).
-
+{% include generated/rocks_db_configurable_configuration.html %}
-In general, configure the task manager to have enough buffers that each logical network connection
-you expect to be open at the same time has a dedicated buffer. A logical network connection exists
-for each point-to-point exchange of data over the network, which typically happens at
-repartitioning or broadcasting steps (shuffle phase). In those, each parallel task inside the
-TaskManager has to be able to talk to all other parallel tasks.
+### Advanced Fault Tolerance Options
-
- Note: Since Flink 1.5, network buffers will always be allocated off-heap, i.e. outside of the JVM heap, irrespective of the value of taskmanager.memory.off-heap. This way, we can pass these buffers directly to the underlying network stack layers.
-
+*These parameters can help with problems related to failover and to components erroneously considering each other as failed.*
-#### Setting Memory Fractions
+{% include generated/expert_fault_tolerance_section.html %}
-Previously, the number of network buffers was set manually which became a quite error-prone task
-(see below). Since Flink 1.3, it is possible to define a fraction of memory that is being used for
-network buffers with the following configuration parameters:
+### Advanced Scheduling Options
-- `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers (DEFAULT: 0.1),
-- `taskmanager.network.memory.min`: Minimum memory size for network buffers (DEFAULT: 64MB),
-- `taskmanager.network.memory.max`: Maximum memory size for network buffers (DEFAULT: 1GB), and
-- `taskmanager.memory.segment-size`: Size of memory buffers used by the memory manager and the
-network stack in bytes (DEFAULT: 32KB).
+*These parameters can help with fine-tuning scheduling for specific situations.*
-#### Setting the Number of Network Buffers directly
+{% include generated/expert_scheduling_section.html %}
-
- Note: This way of configuring the amount of memory used for network buffers is deprecated. Please consider using the method above by defining a fraction of memory to use.
-
+### Advanced High-availability Options
-The required number of buffers on a task manager is
-*total-degree-of-parallelism* (number of targets) \* *intra-node-parallelism* (number of sources in one task manager) \* *n*
-with *n* being a constant that defines how many repartitioning-/broadcasting steps you expect to be
-active at the same time. Since the *intra-node-parallelism* is typically the number of cores, and
-more than 4 repartitioning or broadcasting channels are rarely active in parallel, it frequently
-boils down to
+{% include generated/expert_high_availability_section.html %}
-{% highlight plain %}
-#slots-per-TM^2 * #TMs * 4
-{% endhighlight %}
+### Advanced High-availability ZooKeeper Options
-Where `#slots per TM` are the [number of slots per TaskManager](#configuring-taskmanager-processing-slots) and `#TMs` are the total number of task managers.
+{% include generated/expert_high_availability_zk_section.html %}
-To support, for example, a cluster of 20 8-slot machines, you should use roughly 5000 network
-buffers for optimal throughput.
+### Advanced SSL Security Options
-Each network buffer has by default a size of 32 KiBytes. In the example above, the system would thus
-allocate roughly 300 MiBytes for network buffers.
+{% include generated/expert_security_ssl_section.html %}
-The number and size of network buffers can be configured with the following parameters:
+### Advanced Options for the REST endpoint and Client
-- `taskmanager.network.numberOfBuffers`, and
-- `taskmanager.memory.segment-size`.
+{% include generated/expert_rest_section.html %}
-### Configuring Temporary I/O Directories
+### Advanced Options for Flink Web UI
-Although Flink aims to process as much data in main memory as possible, it is not uncommon that more data needs to be processed than memory is available. Flink's runtime is designed to write temporary data to disk to handle these situations.
+{% include generated/web_configuration.html %}
+
+### Full Flink Master Options
-The `io.tmp.dirs` parameter specifies a list of directories into which Flink writes temporary files. The paths of the directories need to be separated by ':' (colon character). Flink will concurrently write (or read) one temporary file to (from) each configured directory. This way, temporary I/O can be evenly distributed over multiple independent I/O devices such as hard disks to improve performance. To leverage fast I/O devices (e.g., SSD, RAID, NAS), it is possible to specify a directory multiple times.
+**Master / JobManager**
-If the `io.tmp.dirs` parameter is not explicitly specified, Flink writes temporary data to the temporary directory of the operating system, such as */tmp* in Linux systems.
+{% include generated/all_jobmanager_section.html %}
-### Configuring TaskManager processing slots
+**Blob Server**
-Flink executes a program in parallel by splitting it into subtasks and scheduling these subtasks to processing slots.
+The Blob Server is a component in the Flink Master / JobManager. It is used for distribution of objects that are too large to be attached to a RPC message and that benefit from caching (like Jar files or large serialized code objects).
-Each Flink TaskManager provides processing slots in the cluster. The number of slots is typically proportional to the number of available CPU cores __of each__ TaskManager. As a general recommendation, the number of available CPU cores is a good default for `taskmanager.numberOfTaskSlots`.
+{% include generated/blob_server_configuration.html %}
-When starting a Flink application, users can supply the default number of slots to use for that job. The command line value therefore is called `-p` (for parallelism). In addition, it is possible to [set the number of slots in the programming APIs]({{site.baseurl}}/dev/parallel.html) for the whole application and for individual operators.
+**ResourceManager**
+
+These configuration keys control basic Resource Manager behavior, independent of the used resource orchestration management framework (YARN, Mesos, etc.)
+
+{% include generated/resource_manager_configuration.html %}
+
+### Full TaskManagerOptions
+
+{% include generated/all_taskmanager_section.html %}
+
+**Data Transport Network Stack**
+
+These options are for the network stack that handles the streaming and batch data exchanges between TaskManagers.
+
+{% include generated/all_taskmanager_network_section.html %}
+
+### RPC / Akka
+
+Flink uses Akka for RPC between components (JobManager/TaskManager/ResourceManager).
+Flink does not use Akka for data transport.
+
+{% include generated/akka_configuration.html %}
-
+----
+----
-### Configuration Runtime Environment Variables
-You have to set config with prefix `containerized.master.env.` and `containerized.taskmanager.env.` in order to set redefined environment variable in ApplicationMaster and TaskManager.
+# JVM and Logging Options
-- `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
- For example for passing LD_LIBRARY_PATH as an env variable to the AppMaster, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
+{% include generated/environment_configuration.html %}
+
+# Forwarding Environment Variables
+
+You can configure environment variables to be set on the Flink Master and TaskManager processes started on Yarn/Mesos.
+
+ - `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
+ For example for passing LD_LIBRARY_PATH as an env variable to the Master, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
in the flink-conf.yaml.
-- `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
+
+ - `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
+
+----
+----
+
+# Deprecated Options
+
+These options relate to parts of Flink that are not actively developed any more.
+These options may be removed in a future release.
+
+**DataSet API Optimizer**
+
+{% include generated/optimizer_configuration.html %}
+
+**DataSet API Runtime Algorithms**
+
+{% include generated/algorithm_configuration.html %}
+
+**DataSet File Sinks**
+
+{% include generated/deprecated_file_sinks_section.html %}
+
+----
+----
+
+# Backup
+
+#### Execution
+
+{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
+{% include generated/execution_configuration.html %}
+
+#### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+{% include generated/stream_pipeline_configuration.html %}
+
+#### Checkpointing
+
+{% include generated/execution_checkpointing_configuration.html %}
{% top %}
diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md
index cfbd9cc7c141397defe62ae5751538df5b112fb0..23b7ce1dd4057dfe563018f49d7b92ba2e3b5fe2 100644
--- a/docs/ops/config.zh.md
+++ b/docs/ops/config.zh.md
@@ -23,307 +23,415 @@ specific language governing permissions and limitations
under the License.
-->
-**For single-node setups Flink is ready to go out of the box and you don't need to change the default configuration to get started.**
+All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+
+The configuration is parsed and evaluated when the Flink processes are started. Changes to the configuration file require restarting the relevant processes.
The out of the box configuration will use your default Java installation. You can manually set the environment variable `JAVA_HOME` or the configuration key `env.java.home` in `conf/flink-conf.yaml` if you want to manually override the Java runtime to use.
-This page lists the most common options that are typically needed to set up a well performing (distributed) installation. In addition a full list of all available configuration parameters is listed here.
+* This will be replaced by the TOC
+{:toc}
-All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+# Basic Setup
-The system and run scripts parse the config at startup time. Changes to the configuration file require restarting the Flink JobManager and TaskManagers.
+The default configuration supports starting a single-node Flink session cluster without any changes.
+The options in this section are the ones most commonly needed for a basic distributed Flink setup.
-The configuration files for the TaskManagers can be different, Flink does not assume uniform machines in the cluster.
+**Hostnames / Ports**
-* This will be replaced by the TOC
-{:toc}
+These options are only necessary for *standalone* application- or session deployments ([simple standalone]({{site.baseurl}}/ops/deployment/cluster_setup.html) or [Kubernetes]({{site.baseurl}}/ops/deployment/kubernetes.html)).
-## Common Options
+If you use Flink with [Yarn]({{site.baseurl}}/ops/deployment/yarn_setup.html), [Mesos]({{site.baseurl}}/ops/deployment/mesos.html), or the [*active* Kubernetes integration]({{site.baseurl}}/ops/deployment/native_kubernetes.html), the hostnames and ports are automatically discovered.
-{% include generated/common_section.html %}
+ - `rest.address`, `rest.port`: These are used by the client to connect to Flink. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes) service in front of the Flink Master's REST interface.
-## Full Reference
+ - The `jobmanager.rpc.address` (defaults to *"localhost"*) and `jobmanager.rpc.port` (defaults to *6123*) config entries are used by the TaskManager to connect to the JobManager/ResourceManager. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes internal) service for the Flink master (JobManager). This option is ignored on [setups with high-availability]({{site.baseurl}}/ops/jobmanager_high_availability.html) where the leader election mechanism is used to discover this automatically.
-### HDFS
+**Memory Sizes**
-
- Note: These keys are deprecated and it is recommended to configure the Hadoop path with the environment variable HADOOP_CONF_DIR instead.
-
+The default memory sizes support simple streaming/batch applications, but are too low to yield good performance for more complex applications.
+
+ - `jobmanager.heap.size`: Sets the size of the *Flink Master* (JobManager / ResourceManager / Dispatcher) JVM heap.
+ - `taskmanager.memory.process.size`: Total size of the TaskManager process, including everything. Flink will subtract some memory for the JVM's own memory requirements (metaspace and others), and divide and configure the rest automatically between its components (network, managed memory, JVM Heap, etc.).
+
+These value are configured as memory sizes, for example *1536m* or *2g*.
+
+**Parallelism**
+
+ - `taskmanager.numberOfTaskSlots`: The number of slots that a TaskManager offers *(default: 1)*. Each slot can take one task or pipeline.
+ Having multiple slots in a TaskManager can help amortize certain constant overheads (of the JVM, application libraries, or network connections) across parallel tasks or pipelines. See the [Task Slots and Resources]({{site.baseurl}}/concepts/runtime.html#task-slots-and-resources) concepts section for details.
-See also [how to configure Hadoop]({{ site.baseurl }}/ops/deployment/hadoop.html#configure-hadoop).
+ Running more smaller TaskManagers with one slot each is a good starting point and leads to the best isolation between tasks. Dedicating the same resources to fewer larger TaskManagers with more slots can help to increase resource utilization, at the cost of weaker isolation between the tasks (more tasks share the same JVM).
-These parameters configure the default HDFS used by Flink. Setups that do not specify an HDFS configuration have to specify the full path to HDFS files (`hdfs://address:port/path/to/files`) Files will also be written with default HDFS parameters (block size, replication factor).
+ - `parallelism.default`: The default parallelism used when no parallelism is specified anywhere *(default: 1)*.
-- `fs.hdfs.hadoopconf`: The absolute path to the Hadoop File System's (HDFS) configuration **directory** (OPTIONAL VALUE). Specifying this value allows programs to reference HDFS files using short URIs (`hdfs:///path/to/files`, without including the address and port of the NameNode in the file URI). Without this option, HDFS files can be accessed, but require fully qualified URIs like `hdfs://address:port/path/to/files`. This option also causes file writers to pick up the HDFS's default values for block sizes and replication factors. Flink will look for the "core-site.xml" and "hdfs-site.xml" files in the specified directory.
+**Checkpointing**
-- `fs.hdfs.hdfsdefault`: The absolute path of Hadoop's own configuration file "hdfs-default.xml" (DEFAULT: null).
+You can configure checkpointing directly in code within your Flink job or application. Putting these values here in the configuration defines them as defaults in case the application does not configure anything.
-- `fs.hdfs.hdfssite`: The absolute path of Hadoop's own configuration file "hdfs-site.xml" (DEFAULT: null).
+ - `state.backend`: The state backend to use. This defines the data structure mechanism for taking snapshots. Common values are `filesystem` or `rocksdb`.
+ - `state.checkpoints.dir`: The directory to write checkpoints to. This takes a path URI like *s3://mybucket/flink-app/checkpoints* or *hdfs://namenode:port/flink/checkpoints*.
+ - `state.savepoints.dir`: The default directory for savepoints. Takes a path URI, similar to `state.checkpoints.dir`.
-### Core
+**Web UI**
-{% include generated/core_configuration.html %}
+ - `web.submit.enable`: Enables uploading and starting jobs through the Flink UI *(true by default)*. Please note that even when this is disabled, session clusters still accept jobs through REST requests (HTTP calls). This flag only guards the feature to upload jobs in the UI.
+ - `web.upload.dir`: The directory where to store uploaded jobs. Only used when `web.submit.enable` is true.
-### JobManager
+**Other**
-{% include generated/job_manager_configuration.html %}
+ - `io.tmp.dirs`: The directories where Flink puts local data, defaults to the system temp directory (`java.io.tmpdir` property). If a list of directories is configured, Flink will rotate files across the directories.
+
+ The data put in these directories include by default the files created by RocksDB, spilled intermediate results (batch algorithms), and cached jar files.
+
+ This data is NOT relied upon for persistence/recovery, but if this data gets deleted, it typically causes a heavyweight recovery operation. It is hence recommended to set this to a directory that is not automatically periodically purged.
+
+ Yarn, Mesos, and Kubernetes setups automatically configure this value to the local working directories by default.
-### Restart Strategies
+----
+----
-Configuration options to control Flink's restart behaviour in case of job failures.
+# Common Setup Options
+
+*Common options to configure your Flink application or cluster.*
+
+### Hosts and Ports
+
+Options to configure hostnames and ports for the different Flink components.
+
+The JobManager hostname and port are only relevant for standalone setups without high-availability.
+In that setup, the config values are used by the TaskManagers to find (and connect to) the JobManager.
+In all highly-available setups, the TaskManagers discover the JobManager via the High-Availability-Service (for example ZooKeeper).
+
+Setups using resource orchestration frameworks (K8s, Yarn, Mesos) typically use the framework's service discovery facilities.
+
+You do not need to configure any TaskManager hosts and ports, unless the setup requires the use of specific port ranges or specific network interfaces to bind to.
+
+{% include generated/common_host_port_section.html %}
+
+### Fault Tolerance
+
+These configuration options control Flink's restart behaviour in case of failures during the execution.
+By configuring these options in your `flink-conf.yaml`, you define the cluster's default restart strategy.
+
+The default restart strategy will only take effect if no job specific restart strategy has been configured via the `ExecutionConfig`.
{% include generated/restart_strategy_configuration.html %}
-#### Fixed Delay Restart Strategy
+**Fixed Delay Restart Strategy**
{% include generated/fixed_delay_restart_strategy_configuration.html %}
-#### Failure Rate Restart Strategy
+**Failure Rate Restart Strategy**
{% include generated/failure_rate_restart_strategy_configuration.html %}
-### TaskManager
+### Checkpoints and State Backends
-{% include generated/task_manager_configuration.html %}
+These options control the basic setup of state backends and checkpointing behavior.
-For *batch* jobs (or if `taskmanager.memoy.preallocate` is enabled) Flink allocates a fraction of 0.7 of the free memory (total memory configured via taskmanager.heap.size minus memory used for network buffers) for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents OutOfMemoryExceptions because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize the data to convert it into Java objects. All in all, managed memory improves the robustness and speed of the system.
+The options are only relevant for jobs/applications executing in a continuous streaming fashion.
+Jobs/applications executing in a batch fashion do not use state backends and checkpoints, but different internal data structures that are optimized for batch processing.
-The default fraction for managed memory can be adjusted using the taskmanager.memory.fraction parameter. An absolute value may be set using taskmanager.memory.size (overrides the fraction parameter). If desired, the managed memory may be allocated outside the JVM heap. This may improve performance in setups with large memory sizes.
+{% include generated/common_state_backends_section.html %}
-{% include generated/task_manager_memory_configuration.html %}
+### High Availability
-### Distributed Coordination
+High-availability here refers to the ability of the master (JobManager) process to recover from failures.
-{% include generated/cluster_configuration.html %}
+The JobManager ensures consistency during recovery across TaskManagers. For the JobManager itself to recover consistently, an external service must store a minimal amount of recovery metadata (like "ID of last committed checkpoint"), as well as help to elect and lock which JobManager is the leader (to avoid split-brain situations).
-### Distributed Coordination (via Akka)
+{% include generated/common_high_availability_section.html %}
-{% include generated/akka_configuration.html %}
+**Options for high-availability setups with ZooKeeper**
-### REST
+{% include generated/common_high_availability_zk_section.html %}
-{% include generated/rest_configuration.html %}
+### Memory Configuration
-### Blob Server
+These configuration values control the way that TaskManagers and JobManagers use memory.
-{% include generated/blob_server_configuration.html %}
+Flink tries to shield users as much as possible from the complexity of configuring the JVM for data-intensive processing.
+In most cases, users should only need to set the values `taskmanager.memory.process.size` or `taskmanager.memory.flink.size` (depending on how the setup), and possibly adjusting the ratio of JVM heap and Managed Memory via `taskmanager.memory.managed.fraction`. The other options below can be used for performane tuning and fixing memory related errors.
-### Heartbeat Manager
+For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory/mem_setup.html).
-{% include generated/heartbeat_manager_configuration.html %}
+{% include generated/common_memory_section.html %}
-### SSL Settings
+### Miscellaneous Options
-{% include generated/security_configuration.html %}
+{% include generated/common_miscellaneous_section.html %}
-### Netty Shuffle Environment
+----
+----
-{% include generated/netty_shuffle_environment_configuration.html %}
+# Security
-### Network Communication (via Netty)
+Options for configuring Flink's security and secure interaction with external systems.
-These parameters allow for advanced tuning. The default values are sufficient when running concurrent high-throughput jobs on a large cluster.
+### SSL
-{% include generated/network_netty_configuration.html %}
+Flink's network connections can be secured via SSL. Please refer to the [SSL Setup Docs]({{site.baseurl}}/ops/security-ssl.html) for detailed setup guide and background.
-### Web Frontend
+{% include generated/security_ssl_section.html %}
-{% include generated/web_configuration.html %}
-### File Systems
+### Auth with External Systems
-{% include generated/file_system_configuration.html %}
+**ZooKeeper Authentication / Authorization**
-### Compiler/Optimizer
+These options are necessary when connecting to a secured ZooKeeper quorum.
-{% include generated/optimizer_configuration.html %}
+{% include generated/security_auth_zk_section.html %}
-### Runtime Algorithms
+**Kerberos-based Authentication / Authorization**
-{% include generated/algorithm_configuration.html %}
+Please refer to the [Flink and Kerberos Docs]({{site.baseurl}}/ops/security-kerberos.html) for a setup guide and a list of external system to which Flink can authenticate itself via Kerberos.
-### Resource Manager
+{% include generated/security_auth_kerberos_section.html %}
-The configuration keys in this section are independent of the used resource management framework (YARN, Mesos, Standalone, ...)
+----
+----
-{% include generated/resource_manager_configuration.html %}
+# Resource Orchestration Frameworks
-### Shuffle Service
+This section contains options related to integrating Flink with resource orchestration frameworks, like Kubernetes, Yarn, Mesos, etc.
-{% include generated/shuffle_service_configuration.html %}
+Note that is not always necessary to integrate Flink with the resource orchestration framework.
+For example, you can easily deploy Flink applications on Kubernetes without Flink knowing that it runs on Kubernetes (and without specifying any of the Kubernetes config options here.) See [this setup guide]({{site.baseurl}}/ops/deployment/kubernetes.html) for an example.
+
+The options in this section are necessary for setups where Flink itself actively requests and releases resources from the orchestrators.
### YARN
{% include generated/yarn_config_configuration.html %}
+### Kubernetes
+
+{% include generated/kubernetes_config_configuration.html %}
+
### Mesos
{% include generated/mesos_configuration.html %}
-#### Mesos TaskManager
+**Mesos TaskManager**
{% include generated/mesos_task_manager_configuration.html %}
-### High Availability (HA)
+----
+----
-{% include generated/high_availability_configuration.html %}
+# State Backends
-#### ZooKeeper-based HA Mode
+Please refer to the [State Backend Documentation]({{site.baseurl}}/ops/state/state_backends.html) for background on State Backends.
-{% include generated/high_availability_zookeeper_configuration.html %}
+### RocksDB State Backend
-### ZooKeeper Security
+These are the options commonly needed to configure the RocksDB state backend. See the [Advanced RocksDB Backend Section](#advanced-rocksdb-state-backends-options) for options necessary for advanced low level configurations and trouble-shooting.
-{% include generated/zoo_keeper_configuration.html %}
+{% include generated/state_backend_rocksdb_section.html %}
-### Kerberos-based Security
+----
+----
-{% include generated/kerberos_configuration.html %}
+# Metrics
-### Environment
+Please refer to the [metrics system documentation]({{site.baseurl}}/monitoring/metrics.html) for background on Flink's metrics infrastructure.
-{% include generated/environment_configuration.html %}
+{% include generated/metric_configuration.html %}
-### Checkpointing
+### RocksDB Native Metrics
-{% include generated/checkpointing_configuration.html %}
+Flink can report metrics from RocksDB's native code, for applications using the RocksDB state backend.
+The metrics here are scoped to the operators and then further broken down by column family; values are reported as unsigned longs.
-### RocksDB State Backend
+
+ Note: Enabling RocksDB's native metrics may cause degraded performance and should be set carefully.
+
-{% include generated/rocks_db_configuration.html %}
+{% include generated/rocks_db_native_metric_configuration.html %}
-### RocksDB Configurable Options
-Specific RocksDB configurable options, provided by Flink, to create a corresponding `ConfigurableOptionsFactory`.
-And the created one would be used as default `OptionsFactory` in `RocksDBStateBackend`
-unless user define a `OptionsFactory` and set via `RocksDBStateBackend.setOptions(optionsFactory)`
+----
+----
-{% include generated/rocks_db_configurable_configuration.html %}
+# History Server
+
+The history server keeps the information of completed jobs (graphs, runtimes, statistics). To enable it, you have to enable "job archiving" in the JobManager (`jobmanager.archive.fs.dir`).
+
+See the [History Server Docs]({{site.baseurl}}/monitoring/historyserver.html) for details.
+
+{% include generated/history_server_configuration.html %}
+
+----
+----
+
+# Experimental
+
+*Options for experimental features in Flink.*
### Queryable State
+*Queryable State* is an experimental features that gives lets you access Flink's internal state like a key/value store.
+See the [Queryable State Docs]({{site.baseurl}}/dev/stream/state/queryable_state.html) for details.
+
{% include generated/queryable_state_configuration.html %}
-### Metrics
+----
+----
-{% include generated/metric_configuration.html %}
-
-### RocksDB Native Metrics
-Certain RocksDB native metrics may be forwarded to Flink's metrics reporter.
-All native metrics are scoped to operators and then further broken down by column family; values are reported as unsigned longs.
+# Debugging & Expert Tuning
- Note: Enabling native metrics may cause degraded performance and should be set carefully.
+ The options below here are meant for expert users and for fixing/debugging problems. Most setups should not need to configure these options.
-{% include generated/rocks_db_native_metric_configuration.html %}
+### Class Loading
-### History Server
+Flink dynamically loads the code for jobs submitted to a session cluster. In addition, Flink tries to hide many dependencies in the classpath from the application. This helps to reduce dependency conflicts between the application code and the dependencies in the classpath.
-You have to configure `jobmanager.archive.fs.dir` in order to archive terminated jobs and add it to the list of monitored directories via `historyserver.archive.fs.dir` if you want to display them via the HistoryServer's web frontend.
+Please refer to the [Debugging Classloading Docs]({{site.baseurl}}/monitoring/debugging_classloading.html) for details.
-- `jobmanager.archive.fs.dir`: Directory to upload information about terminated jobs to. You have to add this directory to the list of monitored directories of the history server via `historyserver.archive.fs.dir`.
+{% include generated/expert_class_loading_section.html %}
-{% include generated/history_server_configuration.html %}
+### Advanced State Backends Options
-## Legacy
+{% include generated/expert_state_backends_section.html %}
-- `mode`: Execution mode of Flink. Possible values are `legacy` and `new`. In order to start the legacy components, you have to specify `legacy` (DEFAULT: `new`).
+### Advanced RocksDB State Backends Options
-## Background
+Advanced options to tune RocksDB and RocksDB checkpoints.
-### Configuring the Network Buffers
+{% include generated/expert_rocksdb_section.html %}
-If you ever see the Exception `java.io.IOException: Insufficient number of network buffers`, you
-need to adapt the amount of memory used for network buffers in order for your program to run on your
-task managers.
+**RocksDB Configurable Options**
-Network buffers are a critical resource for the communication layers. They are used to buffer
-records before transmission over a network, and to buffer incoming data before dissecting it into
-records and handing them to the application. A sufficient number of network buffers is critical to
-achieve a good throughput.
+These options give fine-grained control over the behavior and resoures of ColumnFamilies.
+With the introduction of `state.backend.rocksdb.memory.managed` and `state.backend.rocksdb.memory.fixed-per-slot` (Apache Flink 1.10), it should be only necessary to use the options here for advanced performance tuning. These options here can also be specified in the application program via `RocksDBStateBackend.setOptions(PptionsFactory)`.
-
-Since Flink 1.3, you may follow the idiom "more is better" without any penalty on the latency (we
-prevent excessive buffering in each outgoing and incoming channel, i.e. *buffer bloat*, by limiting
-the actual number of buffers used by each channel).
-
+{% include generated/rocks_db_configurable_configuration.html %}
-In general, configure the task manager to have enough buffers that each logical network connection
-you expect to be open at the same time has a dedicated buffer. A logical network connection exists
-for each point-to-point exchange of data over the network, which typically happens at
-repartitioning or broadcasting steps (shuffle phase). In those, each parallel task inside the
-TaskManager has to be able to talk to all other parallel tasks.
+### Advanced Fault Tolerance Options
-
- Note: Since Flink 1.5, network buffers will always be allocated off-heap, i.e. outside of the JVM heap, irrespective of the value of taskmanager.memory.off-heap. This way, we can pass these buffers directly to the underlying network stack layers.
-
+*These parameters can help with problems related to failover and to components erroneously considering each other as failed.*
-#### Setting Memory Fractions
+{% include generated/expert_fault_tolerance_section.html %}
-Previously, the number of network buffers was set manually which became a quite error-prone task
-(see below). Since Flink 1.3, it is possible to define a fraction of memory that is being used for
-network buffers with the following configuration parameters:
+### Advanced Scheduling Options
-- `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers (DEFAULT: 0.1),
-- `taskmanager.network.memory.min`: Minimum memory size for network buffers (DEFAULT: 64MB),
-- `taskmanager.network.memory.max`: Maximum memory size for network buffers (DEFAULT: 1GB), and
-- `taskmanager.memory.segment-size`: Size of memory buffers used by the memory manager and the
-network stack in bytes (DEFAULT: 32KB).
+*These parameters can help with fine-tuning scheduling for specific situations.*
-#### Setting the Number of Network Buffers directly
+{% include generated/expert_scheduling_section.html %}
-
- Note: This way of configuring the amount of memory used for network buffers is deprecated. Please consider using the method above by defining a fraction of memory to use.
-
+### Advanced High-availability Options
-The required number of buffers on a task manager is
-*total-degree-of-parallelism* (number of targets) \* *intra-node-parallelism* (number of sources in one task manager) \* *n*
-with *n* being a constant that defines how many repartitioning-/broadcasting steps you expect to be
-active at the same time. Since the *intra-node-parallelism* is typically the number of cores, and
-more than 4 repartitioning or broadcasting channels are rarely active in parallel, it frequently
-boils down to
+{% include generated/expert_high_availability_section.html %}
-{% highlight plain %}
-#slots-per-TM^2 * #TMs * 4
-{% endhighlight %}
+### Advanced High-availability ZooKeeper Options
-Where `#slots per TM` are the [number of slots per TaskManager](#configuring-taskmanager-processing-slots) and `#TMs` are the total number of task managers.
+{% include generated/expert_high_availability_zk_section.html %}
-To support, for example, a cluster of 20 8-slot machines, you should use roughly 5000 network
-buffers for optimal throughput.
+### Advanced SSL Security Options
-Each network buffer has by default a size of 32 KiBytes. In the example above, the system would thus
-allocate roughly 300 MiBytes for network buffers.
+{% include generated/expert_security_ssl_section.html %}
-The number and size of network buffers can be configured with the following parameters:
+### Advanced Options for the REST endpoint and Client
-- `taskmanager.network.numberOfBuffers`, and
-- `taskmanager.memory.segment-size`.
+{% include generated/expert_rest_section.html %}
-### Configuring Temporary I/O Directories
+### Advanced Options for Flink Web UI
-Although Flink aims to process as much data in main memory as possible, it is not uncommon that more data needs to be processed than memory is available. Flink's runtime is designed to write temporary data to disk to handle these situations.
+{% include generated/web_configuration.html %}
+
+### Full Flink Master Options
-The `io.tmp.dirs` parameter specifies a list of directories into which Flink writes temporary files. The paths of the directories need to be separated by ':' (colon character). Flink will concurrently write (or read) one temporary file to (from) each configured directory. This way, temporary I/O can be evenly distributed over multiple independent I/O devices such as hard disks to improve performance. To leverage fast I/O devices (e.g., SSD, RAID, NAS), it is possible to specify a directory multiple times.
+**Master / JobManager**
-If the `io.tmp.dirs` parameter is not explicitly specified, Flink writes temporary data to the temporary directory of the operating system, such as */tmp* in Linux systems.
+{% include generated/all_jobmanager_section.html %}
-### Configuring TaskManager processing slots
+**Blob Server**
-Flink executes a program in parallel by splitting it into subtasks and scheduling these subtasks to processing slots.
+The Blob Server is a component in the Flink Master / JobManager. It is used for distribution of objects that are too large to be attached to a RPC message and that benefit from caching (like Jar files or large serialized code objects).
-Each Flink TaskManager provides processing slots in the cluster. The number of slots is typically proportional to the number of available CPU cores __of each__ TaskManager. As a general recommendation, the number of available CPU cores is a good default for `taskmanager.numberOfTaskSlots`.
+{% include generated/blob_server_configuration.html %}
-When starting a Flink application, users can supply the default number of slots to use for that job. The command line value therefore is called `-p` (for parallelism). In addition, it is possible to [set the number of slots in the programming APIs]({{site.baseurl}}/dev/parallel.html) for the whole application and for individual operators.
+**ResourceManager**
+
+These configuration keys control basic Resource Manager behavior, independent of the used resource orchestration management framework (YARN, Mesos, etc.)
+
+{% include generated/resource_manager_configuration.html %}
+
+### Full TaskManagerOptions
+
+{% include generated/all_taskmanager_section.html %}
+
+**Data Transport Network Stack**
+
+These options are for the network stack that handles the streaming and batch data exchanges between TaskManagers.
+
+{% include generated/all_taskmanager_network_section.html %}
+
+### RPC / Akka
+
+Flink uses Akka for RPC between components (JobManager/TaskManager/ResourceManager).
+Flink does not use Akka for data transport.
+
+{% include generated/akka_configuration.html %}
-
+----
+----
-### Configuration Runtime Environment Variables
-You have to set config with prefix `containerized.master.env.` and `containerized.taskmanager.env.` in order to set redefined environment variable in ApplicationMaster and TaskManager.
+# JVM and Logging Options
-- `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
- For example for passing LD_LIBRARY_PATH as an env variable to the AppMaster, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
+{% include generated/environment_configuration.html %}
+
+# Forwarding Environment Variables
+
+You can configure environment variables to be set on the Flink Master and TaskManager processes started on Yarn/Mesos.
+
+ - `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
+ For example for passing LD_LIBRARY_PATH as an env variable to the Master, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
in the flink-conf.yaml.
-- `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
+
+ - `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
+
+----
+----
+
+# Deprecated Options
+
+These options relate to parts of Flink that are not actively developed any more.
+These options may be removed in a future release.
+
+**DataSet API Optimizer**
+
+{% include generated/optimizer_configuration.html %}
+
+**DataSet API Runtime Algorithms**
+
+{% include generated/algorithm_configuration.html %}
+
+**DataSet File Sinks**
+
+{% include generated/deprecated_file_sinks_section.html %}
+
+----
+----
+
+# Backup
+
+#### Execution
+
+{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
+{% include generated/execution_configuration.html %}
+
+#### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+{% include generated/stream_pipeline_configuration.html %}
+
+#### Checkpointing
+
+{% include generated/execution_checkpointing_configuration.html %}
{% top %}
diff --git a/docs/ops/deployment/aws.md b/docs/ops/deployment/aws.md
deleted file mode 100644
index 50626e28c6940d986a83fcdbb802dc6728c8d205..0000000000000000000000000000000000000000
--- a/docs/ops/deployment/aws.md
+++ /dev/null
@@ -1,71 +0,0 @@
----
-title: "Amazon Web Services (AWS)"
-nav-title: AWS
-nav-parent_id: deployment
-nav-pos: 5
----
-
-
-Amazon Web Services offers cloud computing services on which you can run Flink.
-
-* ToC
-{:toc}
-
-## EMR: Elastic MapReduce
-
-[Amazon Elastic MapReduce](https://aws.amazon.com/elasticmapreduce/) (Amazon EMR) is a web service that makes it easy to quickly setup a Hadoop cluster. This is the **recommended way** to run Flink on AWS as it takes care of setting up everything.
-
-### Standard EMR Installation
-
-Flink is a supported application on Amazon EMR. [Amazon's documentation](http://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-flink.html)
-describes configuring Flink, creating and monitoring a cluster, and working with jobs.
-
-### Custom EMR Installation
-
-Amazon EMR services are regularly updated to new releases but a version of Flink which is not available
-can be manually installed in a stock EMR cluster.
-
-**Create EMR Cluster**
-
-The EMR documentation contains [examples showing how to start an EMR cluster](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-gs-launch-sample-cluster.html). You can follow that guide and install any EMR release. You don't need to install the *All Applications* part of the EMR release, but can stick to *Core Hadoop*.
-
-{% warn Note %}
-Access to S3 buckets requires
-[configuration of IAM roles](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-iam-roles.html)
-when creating an EMR cluster.
-
-**Install Flink on EMR Cluster**
-
-After creating your cluster, you can [connect to the master node](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-connect-master-node.html) and install Flink:
-
-1. Go the [Downloads Page]({{ site.download_url }}) and **download a binary version of Flink matching the Hadoop version** of your EMR cluster, e.g. Hadoop 2.7 for EMR releases 4.3.0, 4.4.0, or 4.5.0.
-2. Make sure all the Hadoop dependencies are in the classpath before you submit any jobs to EMR:
-
-{% highlight bash %}
-export HADOOP_CLASSPATH=`hadoop classpath`
-{% endhighlight %}
-
-3. Extract the Flink distribution and you are ready to deploy [Flink jobs via YARN](yarn_setup.html) after **setting the Hadoop config directory**:
-
-{% highlight bash %}
-HADOOP_CONF_DIR=/etc/hadoop/conf ./bin/flink run -m yarn-cluster examples/streaming/WordCount.jar
-{% endhighlight %}
-
-{% top %}
diff --git a/docs/ops/deployment/aws.zh.md b/docs/ops/deployment/aws.zh.md
deleted file mode 100644
index 50626e28c6940d986a83fcdbb802dc6728c8d205..0000000000000000000000000000000000000000
--- a/docs/ops/deployment/aws.zh.md
+++ /dev/null
@@ -1,71 +0,0 @@
----
-title: "Amazon Web Services (AWS)"
-nav-title: AWS
-nav-parent_id: deployment
-nav-pos: 5
----
-
-
-Amazon Web Services offers cloud computing services on which you can run Flink.
-
-* ToC
-{:toc}
-
-## EMR: Elastic MapReduce
-
-[Amazon Elastic MapReduce](https://aws.amazon.com/elasticmapreduce/) (Amazon EMR) is a web service that makes it easy to quickly setup a Hadoop cluster. This is the **recommended way** to run Flink on AWS as it takes care of setting up everything.
-
-### Standard EMR Installation
-
-Flink is a supported application on Amazon EMR. [Amazon's documentation](http://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-flink.html)
-describes configuring Flink, creating and monitoring a cluster, and working with jobs.
-
-### Custom EMR Installation
-
-Amazon EMR services are regularly updated to new releases but a version of Flink which is not available
-can be manually installed in a stock EMR cluster.
-
-**Create EMR Cluster**
-
-The EMR documentation contains [examples showing how to start an EMR cluster](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-gs-launch-sample-cluster.html). You can follow that guide and install any EMR release. You don't need to install the *All Applications* part of the EMR release, but can stick to *Core Hadoop*.
-
-{% warn Note %}
-Access to S3 buckets requires
-[configuration of IAM roles](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-iam-roles.html)
-when creating an EMR cluster.
-
-**Install Flink on EMR Cluster**
-
-After creating your cluster, you can [connect to the master node](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-connect-master-node.html) and install Flink:
-
-1. Go the [Downloads Page]({{ site.download_url }}) and **download a binary version of Flink matching the Hadoop version** of your EMR cluster, e.g. Hadoop 2.7 for EMR releases 4.3.0, 4.4.0, or 4.5.0.
-2. Make sure all the Hadoop dependencies are in the classpath before you submit any jobs to EMR:
-
-{% highlight bash %}
-export HADOOP_CLASSPATH=`hadoop classpath`
-{% endhighlight %}
-
-3. Extract the Flink distribution and you are ready to deploy [Flink jobs via YARN](yarn_setup.html) after **setting the Hadoop config directory**:
-
-{% highlight bash %}
-HADOOP_CONF_DIR=/etc/hadoop/conf ./bin/flink run -m yarn-cluster examples/streaming/WordCount.jar
-{% endhighlight %}
-
-{% top %}
diff --git a/docs/ops/deployment/cluster_setup.md b/docs/ops/deployment/cluster_setup.md
index 003ef3c785f7c35d4cf08a05dc8384b9a531a239..5efd38d9e75bbbad075b8fb26d205b6515b46104 100644
--- a/docs/ops/deployment/cluster_setup.md
+++ b/docs/ops/deployment/cluster_setup.md
@@ -1,7 +1,7 @@
---
title: "Standalone Cluster"
nav-parent_id: deployment
-nav-pos: 1
+nav-pos: 3
---
-
-
-This documentation provides instructions on how to setup Flink fully automatically with Hadoop 1 or Hadoop 2 on top of a [Google Compute Engine](https://cloud.google.com/compute/) cluster. This is made possible by Google's [bdutil](https://cloud.google.com/hadoop/bdutil) which starts a cluster and deploys Flink with Hadoop. To get started, just follow the steps below.
-
-* This will be replaced by the TOC
-{:toc}
-
-# Prerequisites
-
-## Install Google Cloud SDK
-
-Please follow the instructions on how to setup the [Google Cloud SDK](https://cloud.google.com/sdk/). In particular, make sure to authenticate with Google Cloud using the following command:
-
- gcloud auth login
-
-## Install bdutil
-
-At the moment, there is no bdutil release yet which includes the Flink
-extension. However, you can get the latest version of bdutil with Flink support
-from [GitHub](https://github.com/GoogleCloudPlatform/bdutil):
-
- git clone https://github.com/GoogleCloudPlatform/bdutil.git
-
-After you have downloaded the source, change into the newly created `bdutil` directory and continue with the next steps.
-
-# Deploying Flink on Google Compute Engine
-
-## Set up a bucket
-
-If you have not done so, create a bucket for the bdutil config and staging files. A new bucket can be created with gsutil:
-
- gsutil mb gs://
-
-## Adapt the bdutil config
-
-To deploy Flink with bdutil, adapt at least the following variables in
-bdutil_env.sh.
-
- CONFIGBUCKET=""
- PROJECT=""
- NUM_WORKERS=
-
- # set this to 'n1-standard-2' if you're using the free trial
- GCE_MACHINE_TYPE=""
-
- # for example: "europe-west1-d"
- GCE_ZONE=""
-
-## Adapt the Flink config
-
-bdutil's Flink extension handles the configuration for you. You may additionally adjust configuration variables in `extensions/flink/flink_env.sh`. If you want to make further configuration, please take a look at [configuring Flink](../config.html). You will have to restart Flink after changing its configuration using `bin/stop-cluster` and `bin/start-cluster`.
-
-## Bring up a cluster with Flink
-
-To bring up the Flink cluster on Google Compute Engine, execute:
-
- ./bdutil -e extensions/flink/flink_env.sh deploy
-
-## Run a Flink example job:
-
- ./bdutil shell
- cd /home/hadoop/flink-install/bin
- ./flink run ../examples/batch/WordCount.jar gs://dataflow-samples/shakespeare/othello.txt gs:///output
-
-## Shut down your cluster
-
-Shutting down a cluster is as simple as executing
-
- ./bdutil -e extensions/flink/flink_env.sh delete
-
-{% top %}
diff --git a/docs/ops/deployment/gce_setup.zh.md b/docs/ops/deployment/gce_setup.zh.md
deleted file mode 100644
index 55dc20aeba5c5d0978b7f3fbac44efd2934afef1..0000000000000000000000000000000000000000
--- a/docs/ops/deployment/gce_setup.zh.md
+++ /dev/null
@@ -1,95 +0,0 @@
----
-title: "Google Compute Engine 安装"
-nav-title: Google Compute Engine
-nav-parent_id: deployment
-nav-pos: 6
----
-
-
-
-This documentation provides instructions on how to setup Flink fully automatically with Hadoop 1 or Hadoop 2 on top of a [Google Compute Engine](https://cloud.google.com/compute/) cluster. This is made possible by Google's [bdutil](https://cloud.google.com/hadoop/bdutil) which starts a cluster and deploys Flink with Hadoop. To get started, just follow the steps below.
-
-* This will be replaced by the TOC
-{:toc}
-
-# Prerequisites
-
-## Install Google Cloud SDK
-
-Please follow the instructions on how to setup the [Google Cloud SDK](https://cloud.google.com/sdk/). In particular, make sure to authenticate with Google Cloud using the following command:
-
- gcloud auth login
-
-## Install bdutil
-
-At the moment, there is no bdutil release yet which includes the Flink
-extension. However, you can get the latest version of bdutil with Flink support
-from [GitHub](https://github.com/GoogleCloudPlatform/bdutil):
-
- git clone https://github.com/GoogleCloudPlatform/bdutil.git
-
-After you have downloaded the source, change into the newly created `bdutil` directory and continue with the next steps.
-
-# Deploying Flink on Google Compute Engine
-
-## Set up a bucket
-
-If you have not done so, create a bucket for the bdutil config and staging files. A new bucket can be created with gsutil:
-
- gsutil mb gs://
-
-## Adapt the bdutil config
-
-To deploy Flink with bdutil, adapt at least the following variables in
-bdutil_env.sh.
-
- CONFIGBUCKET=""
- PROJECT=""
- NUM_WORKERS=
-
- # set this to 'n1-standard-2' if you're using the free trial
- GCE_MACHINE_TYPE=""
-
- # for example: "europe-west1-d"
- GCE_ZONE=""
-
-## Adapt the Flink config
-
-bdutil's Flink extension handles the configuration for you. You may additionally adjust configuration variables in `extensions/flink/flink_env.sh`. If you want to make further configuration, please take a look at [configuring Flink](../config.html). You will have to restart Flink after changing its configuration using `bin/stop-cluster` and `bin/start-cluster`.
-
-## Bring up a cluster with Flink
-
-To bring up the Flink cluster on Google Compute Engine, execute:
-
- ./bdutil -e extensions/flink/flink_env.sh deploy
-
-## Run a Flink example job:
-
- ./bdutil shell
- cd /home/hadoop/flink-install/bin
- ./flink run ../examples/batch/WordCount.jar gs://dataflow-samples/shakespeare/othello.txt gs:///output
-
-## Shut down your cluster
-
-Shutting down a cluster is as simple as executing
-
- ./bdutil -e extensions/flink/flink_env.sh delete
-
-{% top %}
diff --git a/docs/ops/deployment/hadoop.md b/docs/ops/deployment/hadoop.md
index bdafad2a99ead535921c2f7c96c059546ef4b0b9..24471853e065576eddb4eeee8497b3e8afe294a4 100644
--- a/docs/ops/deployment/hadoop.md
+++ b/docs/ops/deployment/hadoop.md
@@ -38,13 +38,22 @@ Referencing the HDFS configuration in the [Flink configuration]({{ site.baseurl
Another way to provide the Hadoop configuration is to have it on the class path of the Flink process, see more details below.
-## Adding Hadoop Classpaths
+## Providing Hadoop classes
-The required classes to use Hadoop should be available in the `lib/` folder of the Flink installation
-(on all machines running Flink) unless Flink is built with [Hadoop shaded dependencies]({{ site.baseurl }}/flinkDev/building.html#pre-bundled-versions).
+In order to use Hadoop features (e.g., YARN, HDFS) it is necessary to provide Flink with the required Hadoop classes,
+as these are not bundled by default.
-If putting the files into the directory is not possible, Flink also respects
-the `HADOOP_CLASSPATH` environment variable to add Hadoop jar files to the classpath.
+This can be done by
+1) Adding the Hadoop classpath to Flink
+2) Putting the required jar files into /lib directory of the Flink distribution
+Option 1) requires very little work, integrates nicely with existing Hadoop setups and should be the
+preferred approach.
+However, Hadoop has a large dependency footprint that increases the risk for dependency conflicts to occur.
+If this happens, please refer to option 2).
+
+The following subsections explains these approaches in detail.
+
+### Adding Hadoop Classpaths
Flink will use the environment variable `HADOOP_CLASSPATH` to augment the
classpath that is used when starting Flink components such as the Client,
@@ -66,6 +75,29 @@ in the shell. Note that `hadoop` is the hadoop binary and that `classpath` is an
Putting the Hadoop configuration in the same class path as the Hadoop libraries makes Flink pick up that configuration.
+### Adding Hadoop to /lib
+
+The Flink project releases Hadoop distributions for specific versions, that relocate or exclude several dependencies
+to reduce the risk of dependency clashes.
+These can be found in the [Additional Components]({{ site.download_url }}#additional-components) section of the download page.
+For these versions it is sufficient to download the corresponding `Pre-bundled Hadoop` component and putting it into
+the `/lib` directory of the Flink distribution.
+
+If the used Hadoop version is not listed on the download page (possibly due to being a Vendor-specific version),
+then it is necessary to build [flink-shaded](https://github.com/apache/flink-shaded) against this version.
+You can find the source code for this project in the [Additional Components]({{ site.download_url }}#additional-components) section of the download page.
+
+Note If you want to build `flink-shaded` against a vendor specific Hadoop version, you first have to configure the
+vendor-specific maven repository in your local maven setup as described [here](https://maven.apache.org/guides/mini/guide-multiple-repositories.html).
+
+Run the following command to build and install `flink-shaded` against your desired Hadoop version (e.g., for version `2.6.5-custom`):
+
+{% highlight bash %}
+mvn clean install -Dhadoop.version=2.6.5-custom
+{% endhighlight %}
+
+After this step is complete, put the `flink-shaded-hadoop-2-uber` jar into the `/lib` directory of the Flink distribution.
+
## Running a job locally
To run a job locally as one JVM process using the mini cluster, the required hadoop dependencies have to be explicitly
diff --git a/docs/ops/deployment/hadoop.zh.md b/docs/ops/deployment/hadoop.zh.md
index 34038c97eb11ece4e08cb0320a7fa3af98636609..39c54ccc1c0f527b257ade9f20eae8adf1340b2d 100644
--- a/docs/ops/deployment/hadoop.zh.md
+++ b/docs/ops/deployment/hadoop.zh.md
@@ -38,13 +38,22 @@ Referencing the HDFS configuration in the [Flink configuration]({{ site.baseurl
Another way to provide the Hadoop configuration is to have it on the class path of the Flink process, see more details below.
-## Adding Hadoop Classpaths
+## Providing Hadoop classes
-The required classes to use Hadoop should be available in the `lib/` folder of the Flink installation
-(on all machines running Flink) unless Flink is built with [Hadoop shaded dependencies]({{ site.baseurl }}/flinkDev/building.html#pre-bundled-versions).
+In order to use Hadoop features (e.g., YARN, HDFS) it is necessary to provide Flink with the required Hadoop classes,
+as these are not bundled by default.
-If putting the files into the directory is not possible, Flink also respects
-the `HADOOP_CLASSPATH` environment variable to add Hadoop jar files to the classpath.
+This can be done by
+1) Adding the Hadoop classpath to Flink
+2) Putting the required jar files into /lib directory of the Flink distribution
+Option 1) requires very little work, integrates nicely with existing Hadoop setups and should be the
+preferred approach.
+However, Hadoop has a large dependency footprint that increases the risk for dependency conflicts to occur.
+If this happens, please refer to option 2).
+
+The following subsections explains these approaches in detail.
+
+### Adding Hadoop Classpaths
Flink will use the environment variable `HADOOP_CLASSPATH` to augment the
classpath that is used when starting Flink components such as the Client,
@@ -66,6 +75,29 @@ in the shell. Note that `hadoop` is the hadoop binary and that `classpath` is an
Putting the Hadoop configuration in the same class path as the Hadoop libraries makes Flink pick up that configuration.
+### Adding Hadoop to /lib
+
+The Flink project releases Hadoop distributions for specific versions, that relocate or exclude several dependencies
+to reduce the risk of dependency clashes.
+These can be found in the [Additional Components]({{ site.download_url }}#additional-components) section of the download page.
+For these versions it is sufficient to download the corresponding `Pre-bundled Hadoop` component and putting it into
+the `/lib` directory of the Flink distribution.
+
+If the used Hadoop version is not listed on the download page (possibly due to being a Vendor-specific version),
+then it is necessary to build [flink-shaded](https://github.com/apache/flink-shaded) against this version.
+You can find the source code for this project in the [Additional Components]({{ site.download_url }}#additional-components) section of the download page.
+
+Note If you want to build `flink-shaded` against a vendor specific Hadoop version, you first have to configure the
+vendor-specific maven repository in your local maven setup as described [here](https://maven.apache.org/guides/mini/guide-multiple-repositories.html).
+
+Run the following command to build and install `flink-shaded` against your desired Hadoop version (e.g., for version `2.6.5-custom`):
+
+{% highlight bash %}
+mvn clean install -Dhadoop.version=2.6.5-custom
+{% endhighlight %}
+
+After this step is complete, put the `flink-shaded-hadoop-2-uber` jar into the `/lib` directory of the Flink distribution.
+
## Running a job locally
To run a job locally as one JVM process using the mini cluster, the required hadoop dependencies have to be explicitly
diff --git a/docs/ops/deployment/kubernetes.md b/docs/ops/deployment/kubernetes.md
index 25ffcd3a0237cd6e2ea8fa43ad0d60d2e8bda108..f30f21a22843b498681dbd80494eb987b86d763d 100644
--- a/docs/ops/deployment/kubernetes.md
+++ b/docs/ops/deployment/kubernetes.md
@@ -2,7 +2,7 @@
title: "Kubernetes Setup"
nav-title: Kubernetes
nav-parent_id: deployment
-nav-pos: 4
+nav-pos: 7
---
+Get a local Flink cluster up and running in a few simple steps.
+
* This will be replaced by the TOC
{:toc}
-Get a Flink example program up and running in a few simple steps.
-
## Setup: Download and Start Flink
-Flink runs on __Linux, Mac OS X, and Windows__. To be able to run Flink, the only requirement is to have a working __Java 8.x__ installation. Windows users, please take a look at the [Flink on Windows]({{ site.baseurl }}/getting-started/tutorials/flink_on_windows.html) guide which describes how to run Flink on Windows for local setups.
+Flink runs on __Linux, Mac OS X, and Windows__.
+To be able to run Flink, the only requirement is to have a working __Java 8 or 11__ installation.
You can check the correct installation of Java by issuing the following command:
@@ -48,7 +49,6 @@ Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode)
{% if site.is_stable %}
-
1. Download a binary from the [downloads page](https://flink.apache.org/downloads.html). You can pick
any Scala variant you like. For certain features you may also have to download one of the pre-bundled Hadoop jars
@@ -83,16 +83,40 @@ Clone the source code from one of our [repositories](https://flink.apache.org/co
{% highlight bash %}
$ git clone https://github.com/apache/flink.git
$ cd flink
-$ mvn clean package -DskipTests # this will take up to 10 minutes
-$ cd build-target # this is where Flink is installed to
+
+# Building Flink will take up to 25 minutes
+# You can speed up the build by skipping the
+# web ui by passing the flag '-Pskip-webui-build'
+
+$ mvn clean package -DskipTests -Pfast
+
+# This is where Flink is installed
+$ cd build-target
{% endhighlight %}
{% endif %}
### Start a Local Flink Cluster
+
+Note: The ``bin`` folder of your Java Runtime Environment must be included in Window's ``%PATH%`` variable.
+Follow this [guide](http://www.java.com/en/download/help/path.xml) to add Java to the ``%PATH%`` variable.
+
+{% highlight bash %}
+$ cd flink
+$ cd bin
+$ start-cluster.bat
+{% endhighlight %}
+
+After that, you need to open a second terminal to run jobs using `flink.bat`.
+
+
+
Check the __Dispatcher's web frontend__ at [http://localhost:8081](http://localhost:8081) and make sure everything is up and running. The web frontend should report a single available TaskManager instance.
@@ -114,184 +138,48 @@ INFO ... - Recovering all persisted jobs.
INFO ... - Registering TaskManager ... at ResourceManager
{% endhighlight %}
-## Read the Code
+#### Windows Cygwin Users
-You can find the complete source code for this SocketWindowWordCount example in [scala](https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala) and [java](https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/socket/SocketWindowWordCount.java) on GitHub.
+If you are installing Flink from the git repository and you are using the Windows git shell, Cygwin can produce a failure similar to this one:
-
-
-{% highlight scala %}
-object SocketWindowWordCount {
-
- def main(args: Array[String]) : Unit = {
-
- // the port to connect to
- val port: Int = try {
- ParameterTool.fromArgs(args).getInt("port")
- } catch {
- case e: Exception => {
- System.err.println("No port specified. Please run 'SocketWindowWordCount --port '")
- return
- }
- }
-
- // get the execution environment
- val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
-
- // get input data by connecting to the socket
- val text = env.socketTextStream("localhost", port, '\n')
-
- // parse the data, group it, window it, and aggregate the counts
- val windowCounts = text
- .flatMap { w => w.split("\\s") }
- .map { w => WordWithCount(w, 1) }
- .keyBy("word")
- .timeWindow(Time.seconds(5), Time.seconds(1))
- .sum("count")
-
- // print the results with a single thread, rather than in parallel
- windowCounts.print().setParallelism(1)
-
- env.execute("Socket Window WordCount")
- }
-
- // Data type for words with count
- case class WordWithCount(word: String, count: Long)
-}
-{% endhighlight %}
-
-
-{% highlight java %}
-public class SocketWindowWordCount {
-
- public static void main(String[] args) throws Exception {
-
- // the port to connect to
- final int port;
- try {
- final ParameterTool params = ParameterTool.fromArgs(args);
- port = params.getInt("port");
- } catch (Exception e) {
- System.err.println("No port specified. Please run 'SocketWindowWordCount --port '");
- return;
- }
-
- // get the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // get input data by connecting to the socket
- DataStream text = env.socketTextStream("localhost", port, "\n");
-
- // parse the data, group it, window it, and aggregate the counts
- DataStream windowCounts = text
- .flatMap(new FlatMapFunction() {
- @Override
- public void flatMap(String value, Collector out) {
- for (String word : value.split("\\s")) {
- out.collect(new WordWithCount(word, 1L));
- }
- }
- })
- .keyBy("word")
- .timeWindow(Time.seconds(5), Time.seconds(1))
- .reduce(new ReduceFunction() {
- @Override
- public WordWithCount reduce(WordWithCount a, WordWithCount b) {
- return new WordWithCount(a.word, a.count + b.count);
- }
- });
-
- // print the results with a single thread, rather than in parallel
- windowCounts.print().setParallelism(1);
-
- env.execute("Socket Window WordCount");
- }
-
- // Data type for words with count
- public static class WordWithCount {
-
- public String word;
- public long count;
-
- public WordWithCount() {}
-
- public WordWithCount(String word, long count) {
- this.word = word;
- this.count = count;
- }
-
- @Override
- public String toString() {
- return word + " : " + count;
- }
- }
-}
+{% highlight bash %}
+c:/flink/bin/start-cluster.sh: line 30: $'\r': command not found
{% endhighlight %}
-
-
-## Run the Example
+This error occurs because git is automatically transforming UNIX line endings to Windows style line endings when running in Windows. The problem is that Cygwin can only deal with UNIX style line endings. The solution is to adjust the Cygwin settings to deal with the correct line endings by following these three steps:
-Now, we are going to run this Flink application. It will read text from
-a socket and once every 5 seconds print the number of occurrences of
-each distinct word during the previous 5 seconds, i.e. a tumbling
-window of processing time, as long as words are floating in.
+1. Start a Cygwin shell.
-* First of all, we use **netcat** to start local server via
+2. Determine your home directory by entering
{% highlight bash %}
-$ nc -l 9000
+cd; pwd
{% endhighlight %}
-* Submit the Flink program:
-
-{% highlight bash %}
-$ ./bin/flink run examples/streaming/SocketWindowWordCount.jar --port 9000
-Starting execution of program
-
-{% endhighlight %}
-
- The program connects to the socket and waits for input. You can check the web interface to verify that the job is running as expected:
-
-
-
-
-
-
-
-
-
+ This will return a path under the Cygwin root path.
-* Words are counted in time windows of 5 seconds (processing time, tumbling
- windows) and are printed to `stdout`. Monitor the TaskManager's output file
- and write some text in `nc` (input is sent to Flink line by line after
- hitting ):
+3. Using NotePad, WordPad or a different text editor open the file `.bash_profile` in the home directory and append the following: (If the file does not exist you will have to create it)
{% highlight bash %}
-$ nc -l 9000
-lorem ipsum
-ipsum ipsum ipsum
-bye
+export SHELLOPTS
+set -o igncr
{% endhighlight %}
- The `.out` file will print the counts at the end of each time window as long
- as words are floating in, e.g.:
+Save the file and open a new bash shell.
-{% highlight bash %}
-$ tail -f log/flink-*-taskexecutor-*.out
-lorem : 1
-bye : 1
-ipsum : 4
-{% endhighlight %}
+### Stop a Local Flink Cluster
- To **stop** Flink when you're done type:
+To **stop** Flink when you're done type:
+
+
{% highlight bash %}
$ ./bin/stop-cluster.sh
{% endhighlight %}
-
-## Next Steps
-
-Check out some more [examples]({{ site.baseurl }}/getting-started/examples) to get a better feel for Flink's programming APIs. When you are done with that, go ahead and read the [streaming guide]({{ site.baseurl }}/dev/datastream_api.html).
+
+
+You can terminate the processes via CTRL-C in the spawned shell windows.
+
+
{% top %}
diff --git a/docs/getting-started/tutorials/local_setup.md b/docs/ops/deployment/local.zh.md
similarity index 38%
rename from docs/getting-started/tutorials/local_setup.md
rename to docs/ops/deployment/local.zh.md
index 6d1d09b6ab50885cd35d5834df8c413e1bc234ac..bcbbfdabefc4a3cc05e6a6bb5dc469b2b7b4c394 100644
--- a/docs/getting-started/tutorials/local_setup.md
+++ b/docs/ops/deployment/local.zh.md
@@ -1,8 +1,8 @@
---
-title: "Local Setup Tutorial"
-nav-title: 'Local Setup'
-nav-parent_id: setuptutorials
-nav-pos: 10
+title: "本地集群"
+nav-title: 'Local Cluster'
+nav-parent_id: deployment
+nav-pos: 2
---
+Get a local Flink cluster up and running in a few simple steps.
+
* This will be replaced by the TOC
{:toc}
-Get a Flink example program up and running in a few simple steps.
-
## Setup: Download and Start Flink
-Flink runs on __Linux, Mac OS X, and Windows__. To be able to run Flink, the only requirement is to have a working __Java 8.x__ installation. Windows users, please take a look at the [Flink on Windows]({{ site.baseurl }}/getting-started/tutorials/flink_on_windows.html) guide which describes how to run Flink on Windows for local setups.
+Flink runs on __Linux, Mac OS X, and Windows__.
+To be able to run Flink, the only requirement is to have a working __Java 8 or 11__ installation.
You can check the correct installation of Java by issuing the following command:
@@ -48,7 +49,6 @@ Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode)
{% if site.is_stable %}
-
1. Download a binary from the [downloads page](https://flink.apache.org/downloads.html). You can pick
any Scala variant you like. For certain features you may also have to download one of the pre-bundled Hadoop jars
@@ -83,16 +83,40 @@ Clone the source code from one of our [repositories](https://flink.apache.org/co
{% highlight bash %}
$ git clone https://github.com/apache/flink.git
$ cd flink
-$ mvn clean package -DskipTests # this will take up to 10 minutes
-$ cd build-target # this is where Flink is installed to
+
+# Building Flink will take up to 25 minutes
+# You can speed up the build by skipping the
+# web ui by passing the flag '-Pskip-webui-build'
+
+$ mvn clean package -DskipTests -Pfast
+
+# This is where Flink is installed
+$ cd build-target
{% endhighlight %}
{% endif %}
### Start a Local Flink Cluster
+
+Note: The ``bin`` folder of your Java Runtime Environment must be included in Window's ``%PATH%`` variable.
+Follow this [guide](http://www.java.com/en/download/help/path.xml) to add Java to the ``%PATH%`` variable.
+
+{% highlight bash %}
+$ cd flink
+$ cd bin
+$ start-cluster.bat
+{% endhighlight %}
+
+After that, you need to open a second terminal to run jobs using `flink.bat`.
+
+
+
Check the __Dispatcher's web frontend__ at [http://localhost:8081](http://localhost:8081) and make sure everything is up and running. The web frontend should report a single available TaskManager instance.
@@ -114,184 +138,48 @@ INFO ... - Recovering all persisted jobs.
INFO ... - Registering TaskManager ... at ResourceManager
{% endhighlight %}
-## Read the Code
+#### Windows Cygwin Users
-You can find the complete source code for this SocketWindowWordCount example in [scala](https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala) and [java](https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/socket/SocketWindowWordCount.java) on GitHub.
+If you are installing Flink from the git repository and you are using the Windows git shell, Cygwin can produce a failure similar to this one:
-
-
-{% highlight scala %}
-object SocketWindowWordCount {
-
- def main(args: Array[String]) : Unit = {
-
- // the port to connect to
- val port: Int = try {
- ParameterTool.fromArgs(args).getInt("port")
- } catch {
- case e: Exception => {
- System.err.println("No port specified. Please run 'SocketWindowWordCount --port '")
- return
- }
- }
-
- // get the execution environment
- val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
-
- // get input data by connecting to the socket
- val text = env.socketTextStream("localhost", port, '\n')
-
- // parse the data, group it, window it, and aggregate the counts
- val windowCounts = text
- .flatMap { w => w.split("\\s") }
- .map { w => WordWithCount(w, 1) }
- .keyBy("word")
- .timeWindow(Time.seconds(5), Time.seconds(1))
- .sum("count")
-
- // print the results with a single thread, rather than in parallel
- windowCounts.print().setParallelism(1)
-
- env.execute("Socket Window WordCount")
- }
-
- // Data type for words with count
- case class WordWithCount(word: String, count: Long)
-}
-{% endhighlight %}
-
-
-{% highlight java %}
-public class SocketWindowWordCount {
-
- public static void main(String[] args) throws Exception {
-
- // the port to connect to
- final int port;
- try {
- final ParameterTool params = ParameterTool.fromArgs(args);
- port = params.getInt("port");
- } catch (Exception e) {
- System.err.println("No port specified. Please run 'SocketWindowWordCount --port '");
- return;
- }
-
- // get the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // get input data by connecting to the socket
- DataStream text = env.socketTextStream("localhost", port, "\n");
-
- // parse the data, group it, window it, and aggregate the counts
- DataStream windowCounts = text
- .flatMap(new FlatMapFunction() {
- @Override
- public void flatMap(String value, Collector out) {
- for (String word : value.split("\\s")) {
- out.collect(new WordWithCount(word, 1L));
- }
- }
- })
- .keyBy("word")
- .timeWindow(Time.seconds(5), Time.seconds(1))
- .reduce(new ReduceFunction() {
- @Override
- public WordWithCount reduce(WordWithCount a, WordWithCount b) {
- return new WordWithCount(a.word, a.count + b.count);
- }
- });
-
- // print the results with a single thread, rather than in parallel
- windowCounts.print().setParallelism(1);
-
- env.execute("Socket Window WordCount");
- }
-
- // Data type for words with count
- public static class WordWithCount {
-
- public String word;
- public long count;
-
- public WordWithCount() {}
-
- public WordWithCount(String word, long count) {
- this.word = word;
- this.count = count;
- }
-
- @Override
- public String toString() {
- return word + " : " + count;
- }
- }
-}
+{% highlight bash %}
+c:/flink/bin/start-cluster.sh: line 30: $'\r': command not found
{% endhighlight %}
-
-
-## Run the Example
+This error occurs because git is automatically transforming UNIX line endings to Windows style line endings when running in Windows. The problem is that Cygwin can only deal with UNIX style line endings. The solution is to adjust the Cygwin settings to deal with the correct line endings by following these three steps:
-Now, we are going to run this Flink application. It will read text from
-a socket and once every 5 seconds print the number of occurrences of
-each distinct word during the previous 5 seconds, i.e. a tumbling
-window of processing time, as long as words are floating in.
+1. Start a Cygwin shell.
-* First of all, we use **netcat** to start local server via
+2. Determine your home directory by entering
{% highlight bash %}
-$ nc -l 9000
+cd; pwd
{% endhighlight %}
-* Submit the Flink program:
-
-{% highlight bash %}
-$ ./bin/flink run examples/streaming/SocketWindowWordCount.jar --port 9000
-Starting execution of program
-
-{% endhighlight %}
-
- The program connects to the socket and waits for input. You can check the web interface to verify that the job is running as expected:
-
-
-
-
-
-
-
-
-
+ This will return a path under the Cygwin root path.
-* Words are counted in time windows of 5 seconds (processing time, tumbling
- windows) and are printed to `stdout`. Monitor the TaskManager's output file
- and write some text in `nc` (input is sent to Flink line by line after
- hitting ):
+3. Using NotePad, WordPad or a different text editor open the file `.bash_profile` in the home directory and append the following: (If the file does not exist you will have to create it)
{% highlight bash %}
-$ nc -l 9000
-lorem ipsum
-ipsum ipsum ipsum
-bye
+export SHELLOPTS
+set -o igncr
{% endhighlight %}
- The `.out` file will print the counts at the end of each time window as long
- as words are floating in, e.g.:
+Save the file and open a new bash shell.
-{% highlight bash %}
-$ tail -f log/flink-*-taskexecutor-*.out
-lorem : 1
-bye : 1
-ipsum : 4
-{% endhighlight %}
+### Stop a Local Flink Cluster
- To **stop** Flink when you're done type:
+To **stop** Flink when you're done type:
+
+
{% highlight bash %}
$ ./bin/stop-cluster.sh
{% endhighlight %}
-
-## Next Steps
-
-Check out some more [examples]({{ site.baseurl }}/getting-started/examples) to get a better feel for Flink's programming APIs. When you are done with that, go ahead and read the [streaming guide]({{ site.baseurl }}/dev/datastream_api.html).
+
+
+You can terminate the processes via CTRL-C in the spawned shell windows.
+
+
{% top %}
diff --git a/docs/ops/deployment/mapr_setup.md b/docs/ops/deployment/mapr_setup.md
deleted file mode 100644
index 37b192f94b7b8dfc69b3e1b4d8a717b964099871..0000000000000000000000000000000000000000
--- a/docs/ops/deployment/mapr_setup.md
+++ /dev/null
@@ -1,134 +0,0 @@
----
-title: "MapR Setup"
-nav-title: MapR
-nav-parent_id: deployment
-nav-pos: 7
----
-
-
-This documentation provides instructions on how to prepare Flink for YARN
-executions on a [MapR](https://mapr.com/) cluster.
-
-* This will be replaced by the TOC
-{:toc}
-
-## Running Flink on YARN with MapR
-
-The instructions below assume MapR version 5.2.0. They will guide you
-to be able to start submitting [Flink on YARN]({{ site.baseurl }}/ops/deployment/yarn_setup.html)
-jobs or sessions to a MapR cluster.
-
-### Building Flink for MapR
-
-In order to run Flink on MapR, Flink needs to be built with MapR's own
-Hadoop and Zookeeper distribution. Simply build Flink using Maven with
-the following command from the project root directory:
-
-{% highlight bash %}
-mvn clean install -DskipTests -Pvendor-repos,mapr \
- -Dhadoop.version=2.7.0-mapr-1607 \
- -Dzookeeper.version=3.4.5-mapr-1604
-{% endhighlight %}
-
-The `vendor-repos` build profile adds MapR's repository to the build so that
-MapR's Hadoop / Zookeeper dependencies can be fetched. The `mapr` build
-profile additionally resolves some dependency clashes between MapR and
-Flink, as well as ensuring that the native MapR libraries on the cluster
-nodes are used. Both profiles must be activated.
-
-By default the `mapr` profile builds with Hadoop / Zookeeper dependencies
-for MapR version 5.2.0, so you don't need to explicitly override
-the `hadoop.version` and `zookeeper.version` properties.
-For different MapR versions, simply override these properties to appropriate
-values. The corresponding Hadoop / Zookeeper distributions for each MapR version
-can be found on MapR documentations such as
-[here](http://maprdocs.mapr.com/home/DevelopmentGuide/MavenArtifacts.html).
-
-### Job Submission Client Setup
-
-The client submitting Flink jobs to MapR also needs to be prepared with the below setups.
-
-Ensure that MapR's JAAS config file is picked up to avoid login failures:
-
-{% highlight bash %}
-export JVM_ARGS=-Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf
-{% endhighlight %}
-
-Make sure that the `yarn.nodemanager.resource.cpu-vcores` property is set in `yarn-site.xml`:
-
-{% highlight xml %}
-
-
-
-...
-
-
- yarn.nodemanager.resource.cpu-vcores
- ...
-
-
-...
-
-{% endhighlight %}
-
-Also remember to set the `YARN_CONF_DIR` or `HADOOP_CONF_DIR` environment
-variables to the path where `yarn-site.xml` is located:
-
-{% highlight bash %}
-export YARN_CONF_DIR=/opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/
-export HADOOP_CONF_DIR=/opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/
-{% endhighlight %}
-
-Make sure that the MapR native libraries are picked up in the classpath:
-
-{% highlight bash %}
-export FLINK_CLASSPATH=/opt/mapr/lib/*
-{% endhighlight %}
-
-If you'll be starting Flink on YARN sessions with `yarn-session.sh`, the
-below is also required:
-
-{% highlight bash %}
-export CC_CLASSPATH=/opt/mapr/lib/*
-{% endhighlight %}
-
-## Running Flink with a Secured MapR Cluster
-
-*Note: In Flink 1.2.0, Flink's Kerberos authentication for YARN execution has
-a bug that forbids it to work with MapR Security. Please upgrade to later Flink
-versions in order to use Flink with a secured MapR cluster. For more details,
-please see [FLINK-5949](https://issues.apache.org/jira/browse/FLINK-5949).*
-
-Flink's [Kerberos authentication]({{ site.baseurl }}/ops/security-kerberos.html) is independent of
-[MapR's Security authentication](http://maprdocs.mapr.com/home/SecurityGuide/Configuring-MapR-Security.html).
-With the above build procedures and environment variable setups, Flink
-does not require any additional configuration to work with MapR Security.
-
-Users simply need to login by using MapR's `maprlogin` authentication
-utility. Users that haven't acquired MapR login credentials would not be
-able to submit Flink jobs, erroring with:
-
-{% highlight plain %}
-java.lang.Exception: unable to establish the security context
-Caused by: o.a.f.r.security.modules.SecurityModule$SecurityInstallException: Unable to set the Hadoop login user
-Caused by: java.io.IOException: failure to login: Unable to obtain MapR credentials
-{% endhighlight %}
-
-{% top %}
diff --git a/docs/ops/deployment/mapr_setup.zh.md b/docs/ops/deployment/mapr_setup.zh.md
deleted file mode 100644
index e8d5ea2e02e6d74d9f48f467e8d617b402b62e6a..0000000000000000000000000000000000000000
--- a/docs/ops/deployment/mapr_setup.zh.md
+++ /dev/null
@@ -1,134 +0,0 @@
----
-title: "MapR 安装"
-nav-title: MapR
-nav-parent_id: deployment
-nav-pos: 7
----
-
-
-This documentation provides instructions on how to prepare Flink for YARN
-executions on a [MapR](https://mapr.com/) cluster.
-
-* This will be replaced by the TOC
-{:toc}
-
-## Running Flink on YARN with MapR
-
-The instructions below assume MapR version 5.2.0. They will guide you
-to be able to start submitting [Flink on YARN]({{ site.baseurl }}/ops/deployment/yarn_setup.html)
-jobs or sessions to a MapR cluster.
-
-### Building Flink for MapR
-
-In order to run Flink on MapR, Flink needs to be built with MapR's own
-Hadoop and Zookeeper distribution. Simply build Flink using Maven with
-the following command from the project root directory:
-
-{% highlight bash %}
-mvn clean install -DskipTests -Pvendor-repos,mapr \
- -Dhadoop.version=2.7.0-mapr-1607 \
- -Dzookeeper.version=3.4.5-mapr-1604
-{% endhighlight %}
-
-The `vendor-repos` build profile adds MapR's repository to the build so that
-MapR's Hadoop / Zookeeper dependencies can be fetched. The `mapr` build
-profile additionally resolves some dependency clashes between MapR and
-Flink, as well as ensuring that the native MapR libraries on the cluster
-nodes are used. Both profiles must be activated.
-
-By default the `mapr` profile builds with Hadoop / Zookeeper dependencies
-for MapR version 5.2.0, so you don't need to explicitly override
-the `hadoop.version` and `zookeeper.version` properties.
-For different MapR versions, simply override these properties to appropriate
-values. The corresponding Hadoop / Zookeeper distributions for each MapR version
-can be found on MapR documentations such as
-[here](http://maprdocs.mapr.com/home/DevelopmentGuide/MavenArtifacts.html).
-
-### Job Submission Client Setup
-
-The client submitting Flink jobs to MapR also needs to be prepared with the below setups.
-
-Ensure that MapR's JAAS config file is picked up to avoid login failures:
-
-{% highlight bash %}
-export JVM_ARGS=-Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf
-{% endhighlight %}
-
-Make sure that the `yarn.nodemanager.resource.cpu-vcores` property is set in `yarn-site.xml`:
-
-{% highlight xml %}
-
-
-
-...
-
-
- yarn.nodemanager.resource.cpu-vcores
- ...
-
-
-...
-
-{% endhighlight %}
-
-Also remember to set the `YARN_CONF_DIR` or `HADOOP_CONF_DIR` environment
-variables to the path where `yarn-site.xml` is located:
-
-{% highlight bash %}
-export YARN_CONF_DIR=/opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/
-export HADOOP_CONF_DIR=/opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/
-{% endhighlight %}
-
-Make sure that the MapR native libraries are picked up in the classpath:
-
-{% highlight bash %}
-export FLINK_CLASSPATH=/opt/mapr/lib/*
-{% endhighlight %}
-
-If you'll be starting Flink on YARN sessions with `yarn-session.sh`, the
-below is also required:
-
-{% highlight bash %}
-export CC_CLASSPATH=/opt/mapr/lib/*
-{% endhighlight %}
-
-## Running Flink with a Secured MapR Cluster
-
-*Note: In Flink 1.2.0, Flink's Kerberos authentication for YARN execution has
-a bug that forbids it to work with MapR Security. Please upgrade to later Flink
-versions in order to use Flink with a secured MapR cluster. For more details,
-please see [FLINK-5949](https://issues.apache.org/jira/browse/FLINK-5949).*
-
-Flink's [Kerberos authentication]({{ site.baseurl }}/ops/security-kerberos.html) is independent of
-[MapR's Security authentication](http://maprdocs.mapr.com/home/SecurityGuide/Configuring-MapR-Security.html).
-With the above build procedures and environment variable setups, Flink
-does not require any additional configuration to work with MapR Security.
-
-Users simply need to login by using MapR's `maprlogin` authentication
-utility. Users that haven't acquired MapR login credentials would not be
-able to submit Flink jobs, erroring with:
-
-{% highlight plain %}
-java.lang.Exception: unable to establish the security context
-Caused by: o.a.f.r.security.modules.SecurityModule$SecurityInstallException: Unable to set the Hadoop login user
-Caused by: java.io.IOException: failure to login: Unable to obtain MapR credentials
-{% endhighlight %}
-
-{% top %}
diff --git a/docs/ops/deployment/mesos.md b/docs/ops/deployment/mesos.md
index 476e16ede6fb1d4ec143af44b844f043f607a439..1df18384fc0358670228a3f60c14570eec863dbc 100644
--- a/docs/ops/deployment/mesos.md
+++ b/docs/ops/deployment/mesos.md
@@ -2,7 +2,7 @@
title: "Mesos Setup"
nav-title: Mesos
nav-parent_id: deployment
-nav-pos: 3
+nav-pos: 5
---
+
+This page describes how to deploy a Flink session cluster natively on [Kubernetes](https://kubernetes.io).
+
+* This will be replaced by the TOC
+{:toc}
+
+
+Flink's native Kubernetes integration is still experimental. There may be changes in the configuration and CLI flags in latter versions. Job clusters are not yet supported.
+
+
+## Requirements
+
+- Kubernetes 1.9 or above.
+- KubeConfig, which has access to list, create, delete pods and services, configurable via `~/.kube/config`. You can verify permissions by running `kubectl auth can-i pods`.
+- Kubernetes DNS enabled.
+- A service Account with [RBAC](#rbac) permissions to create, delete pods.
+
+## Flink Kubernetes Session
+
+### Start Flink Session
+
+Follow these instructions to start a Flink Session within your Kubernetes cluster.
+
+A session will start all required Flink services (JobManager and TaskManagers) so that you can submit programs to the cluster.
+Note that you can run multiple programs per session.
+
+{% highlight bash %}
+$ ./bin/kubernetes-session.sh
+{% endhighlight %}
+
+All the Kubernetes configuration options can be found in our [configuration guide]({{ site.baseurl }}/ops/config.html#kubernetes).
+
+**Example**: Issue the following command to start a session cluster with 4 GB of memory and 2 CPUs with 4 slots per TaskManager:
+
+In this example we override the `resourcemanager.taskmanager-timeout` setting to make
+the pods with task managers remain for a longer period than the default of 30 seconds.
+Although this setting may cause more cloud cost it has the effect that starting new jobs is in some scenarios
+faster and during development you have more time to inspect the logfiles of your job.
+
+{% highlight bash %}
+./bin/kubernetes-session.sh \
+ -Dkubernetes.cluster-id= \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dresourcemanager.taskmanager-timeout=3600000
+{% endhighlight %}
+
+The system will use the configuration in `conf/flink-conf.yaml`.
+Please follow our [configuration guide]({{ site.baseurl }}/ops/config.html) if you want to change something.
+
+If you do not specify a particular name for your session by `kubernetes.cluster-id`, the Flink client will generate a UUID name.
+
+### Submitting jobs to an existing Session
+
+Use the following command to submit a Flink Job to the Kubernetes cluster.
+
+{% highlight bash %}
+$ ./bin/flink run -d -e kubernetes-session -Dkubernetes.cluster-id= examples/streaming/WindowJoin.jar
+{% endhighlight %}
+
+### Accessing Job Manager UI
+
+There are several ways to expose a Service onto an external (outside of your cluster) IP address.
+This can be configured using `kubernetes.service.exposed.type`.
+
+- `ClusterIP`: Exposes the service on a cluster-internal IP.
+The Service is only reachable within the cluster. If you want to access the Job Manager ui or submit job to the existing session, you need to start a local proxy.
+You can then use `localhost:8081` to submit a Flink job to the session or view the dashboard.
+
+{% highlight bash %}
+$ kubectl port-forward service/ 8081
+{% endhighlight %}
+
+- `NodePort`: Exposes the service on each Node’s IP at a static port (the `NodePort`). `:` could be used to contact the Job Manager Service. `NodeIP` could be easily replaced with Kubernetes ApiServer address.
+You could find it in your kube config file.
+
+- `LoadBalancer`: Default value, exposes the service externally using a cloud provider’s load balancer.
+Since the cloud provider and Kubernetes needs some time to prepare the load balancer, you may get a `NodePort` JobManager Web Interface in the client log.
+You can use `kubectl get services/` to get EXTERNAL-IP and then construct the load balancer JobManager Web Interface manually `http://:8081`.
+
+- `ExternalName`: Map a service to a DNS name, not supported in current version.
+
+Please reference the official documentation on [publishing services in Kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) for more information.
+
+### Attach to an existing Session
+
+The Kubernetes session is started in detached mode by default, meaning the Flink client will exit after submitting all the resources to the Kubernetes cluster. Use the following command to attach to an existing session.
+
+{% highlight bash %}
+$ ./bin/kubernetes-session.sh -Dkubernetes.cluster-id= -Dexecution.attached=true
+{% endhighlight %}
+
+### Stop Flink Session
+
+To stop a Flink Kubernetes session, attach the Flink client to the cluster and type `stop`.
+
+{% highlight bash %}
+$ echo 'stop' | ./bin/kubernetes-session.sh -Dkubernetes.cluster-id= -Dexecution.attached=true
+{% endhighlight %}
+
+#### Manual Resource Cleanup
+
+Flink uses [Kubernetes ownerReference's](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/) to cleanup all cluster components.
+All the Flink created resources, including `ConfigMap`, `Service`, `Deployment`, `Pod`, have been set the ownerReference to `service/`.
+When the service is deleted, all other resource will be deleted automatically.
+
+{% highlight bash %}
+$ kubectl delete service/
+{% endhighlight %}
+
+## Log Files
+
+By default, the JobManager and TaskManager only store logs under `/opt/flink/log` in each pod.
+If you want to use `kubectl logs ` to view the logs, you must perform the following:
+
+1. Add a new appender to the log4j.properties in the Flink client.
+2. Add the following 'appenderRef' the rootLogger in log4j.properties `rootLogger.appenderRef.console.ref = ConsoleAppender`.
+3. Remove the redirect args by adding config option `-Dkubernetes.container-start-command-template="%java% %classpath% %jvmmem% %jvmopts% %logging% %class% %args%"`.
+4. Stop and start your session again. Now you could use `kubectl logs` to view your logs.
+
+{% highlight bash %}
+# Log all infos to the console
+appender.console.name = ConsoleAppender
+appender.console.type = CONSOLE
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+{% endhighlight %}
+
+If the pod is running, you can use `kubectl exec -it bash` to tunnel in and view the logs or debug the process.
+
+## Kubernetes concepts
+
+### Namespaces
+
+[Namespaces in Kubernetes](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) are a way to divide cluster resources between multiple users (via resource quota).
+It is similar to the queue concept in Yarn cluster. Flink on Kubernetes can use namespaces to launch Flink clusters.
+The namespace can be specified using the `-Dkubernetes.namespace=default` argument when starting a Flink cluster.
+
+[ResourceQuota](https://kubernetes.io/docs/concepts/policy/resource-quotas/) provides constraints that limit aggregate resource consumption per namespace.
+It can limit the quantity of objects that can be created in a namespace by type, as well as the total amount of compute resources that may be consumed by resources in that project.
+
+### RBAC
+
+Role-based access control ([RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)) is a method of regulating access to compute or network resources based on the roles of individual users within an enterprise.
+Users can configure RBAC roles and service accounts used by Flink JobManager to access the Kubernetes API server within the Kubernetes cluster.
+
+Every namespace has a default service account, however, the `default` service account may not have the permission to create or delete pods within the Kubernetes cluster.
+Users may need to update the permission of `default` service account or specify another service account that has the right role bound.
+
+{% highlight bash %}
+$ kubectl create clusterrolebinding flink-role-binding-default --clusterrole=edit --serviceaccount=default:default
+{% endhighlight %}
+
+If you do not want to use `default` service account, use the following command to create a new `flink` service account and set the role binding.
+Then use the config option `-Dkubernetes.jobmanager.service-account=flink` to make the JobManager pod using the `flink` service account to create and delete TaskManager pods.
+
+{% highlight bash %}
+$ kubectl create serviceaccount flink
+$ kubectl create clusterrolebinding flink-role-binding-flink --clusterrole=edit --serviceaccount=default:flink
+{% endhighlight %}
+
+Please reference the official Kubernetes documentation on [RBAC Authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) for more information.
+
+## Background / Internals
+
+This section briefly explains how Flink and Kubernetes interact.
+
+
+
+When creating a Flink Kubernetes session cluster, the Flink client will first connect to the Kubernetes ApiServer to submit the cluster description, including ConfigMap spec, Job Manager Service spec, Job Manager Deployment spec and Owner Reference.
+Kubernetes will then create the Flink master deployment, during which time the Kubelet will pull the image, prepare and mount the volume, and then execute the start command.
+After the master pod has launched, the Dispatcher and KubernetesResourceManager are available and the cluster is ready to accept one or more jobs.
+
+When users submit jobs through the Flink client, the job graph will be generated by the client and uploaded along with users jars to the Dispatcher.
+A JobMaster for that Job will be then be spawned.
+
+The JobMaster requests resources, known as slots, from the KubernetesResourceManager.
+If no slots are available, the resource manager will bring up TaskManager pods and registering them with the cluster.
+
+{% top %}
diff --git a/docs/ops/deployment/native_kubernetes.zh.md b/docs/ops/deployment/native_kubernetes.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..10d47e02a7b84abfb84367013c789521649938cf
--- /dev/null
+++ b/docs/ops/deployment/native_kubernetes.zh.md
@@ -0,0 +1,206 @@
+---
+title: "原生 Kubernetes 设置"
+nav-title: Native Kubernetes
+nav-parent_id: deployment
+is_beta: true
+nav-pos: 7
+---
+
+
+This page describes how to deploy a Flink session cluster natively on [Kubernetes](https://kubernetes.io).
+
+* This will be replaced by the TOC
+{:toc}
+
+
+Flink's native Kubernetes integration is still experimental. There may be changes in the configuration and CLI flags in latter versions. Job clusters are not yet supported.
+
+
+## Requirements
+
+- Kubernetes 1.9 or above.
+- KubeConfig, which has access to list, create, delete pods and services, configurable via `~/.kube/config`. You can verify permissions by running `kubectl auth can-i pods`.
+- Kubernetes DNS enabled.
+- A service Account with [RBAC](#rbac) permissions to create, delete pods.
+
+## Flink Kubernetes Session
+
+### Start Flink Session
+
+Follow these instructions to start a Flink Session within your Kubernetes cluster.
+
+A session will start all required Flink services (JobManager and TaskManagers) so that you can submit programs to the cluster.
+Note that you can run multiple programs per session.
+
+{% highlight bash %}
+$ ./bin/kubernetes-session.sh
+{% endhighlight %}
+
+All the Kubernetes configuration options can be found in our [configuration guide]({{ site.baseurl }}/zh/ops/config.html#kubernetes).
+
+**Example**: Issue the following command to start a session cluster with 4 GB of memory and 2 CPUs with 4 slots per TaskManager:
+
+In this example we override the `resourcemanager.taskmanager-timeout` setting to make
+the pods with task managers remain for a longer period than the default of 30 seconds.
+Although this setting may cause more cloud cost it has the effect that starting new jobs is in some scenarios
+faster and during development you have more time to inspect the logfiles of your job.
+
+{% highlight bash %}
+./bin/kubernetes-session.sh \
+ -Dkubernetes.cluster-id= \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dresourcemanager.taskmanager-timeout=3600000
+{% endhighlight %}
+
+The system will use the configuration in `conf/flink-conf.yaml`.
+Please follow our [configuration guide]({{ site.baseurl }}/zh/ops/config.html) if you want to change something.
+
+If you do not specify a particular name for your session by `kubernetes.cluster-id`, the Flink client will generate a UUID name.
+
+### Submitting jobs to an existing Session
+
+Use the following command to submit a Flink Job to the Kubernetes cluster.
+
+{% highlight bash %}
+$ ./bin/flink run -d -e kubernetes-session -Dkubernetes.cluster-id= examples/streaming/WindowJoin.jar
+{% endhighlight %}
+
+### Accessing Job Manager UI
+
+There are several ways to expose a Service onto an external (outside of your cluster) IP address.
+This can be configured using `kubernetes.service.exposed.type`.
+
+- `ClusterIP`: Exposes the service on a cluster-internal IP.
+The Service is only reachable within the cluster. If you want to access the Job Manager ui or submit job to the existing session, you need to start a local proxy.
+You can then use `localhost:8081` to submit a Flink job to the session or view the dashboard.
+
+{% highlight bash %}
+$ kubectl port-forward service/ 8081
+{% endhighlight %}
+
+- `NodePort`: Exposes the service on each Node’s IP at a static port (the `NodePort`). `:` could be used to contact the Job Manager Service. `NodeIP` could be easily replaced with Kubernetes ApiServer address.
+You could find it in your kube config file.
+
+- `LoadBalancer`: Default value, exposes the service externally using a cloud provider’s load balancer.
+Since the cloud provider and Kubernetes needs some time to prepare the load balancer, you may get a `NodePort` JobManager Web Interface in the client log.
+You can use `kubectl get services/` to get EXTERNAL-IP and then construct the load balancer JobManager Web Interface manually `http://:8081`.
+
+- `ExternalName`: Map a service to a DNS name, not supported in current version.
+
+Please reference the official documentation on [publishing services in Kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) for more information.
+
+### Attach to an existing Session
+
+The Kubernetes session is started in detached mode by default, meaning the Flink client will exit after submitting all the resources to the Kubernetes cluster. Use the following command to attach to an existing session.
+
+{% highlight bash %}
+$ ./bin/kubernetes-session.sh -Dkubernetes.cluster-id= -Dexecution.attached=true
+{% endhighlight %}
+
+### Stop Flink Session
+
+To stop a Flink Kubernetes session, attach the Flink client to the cluster and type `stop`.
+
+{% highlight bash %}
+$ echo 'stop' | ./bin/kubernetes-session.sh -Dkubernetes.cluster-id= -Dexecution.attached=true
+{% endhighlight %}
+
+#### Manual Resource Cleanup
+
+Flink uses [Kubernetes ownerReference's](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/) to cleanup all cluster components.
+All the Flink created resources, including `ConfigMap`, `Service`, `Deployment`, `Pod`, have been set the ownerReference to `service/`.
+When the service is deleted, all other resource will be deleted automatically.
+
+{% highlight bash %}
+$ kubectl delete service/
+{% endhighlight %}
+
+## Log Files
+
+By default, the JobManager and TaskManager only store logs under `/opt/flink/log` in each pod.
+If you want to use `kubectl logs ` to view the logs, you must perform the following:
+
+1. Add a new appender to the log4j.properties in the Flink client.
+2. Add the following 'appenderRef' the rootLogger in log4j.properties `rootLogger.appenderRef.console.ref = ConsoleAppender`.
+3. Remove the redirect args by adding config option `-Dkubernetes.container-start-command-template="%java% %classpath% %jvmmem% %jvmopts% %logging% %class% %args%"`.
+4. Stop and start your session again. Now you could use `kubectl logs` to view your logs.
+
+{% highlight bash %}
+# Log all infos to the console
+appender.console.name = ConsoleAppender
+appender.console.type = CONSOLE
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+{% endhighlight %}
+
+If the pod is running, you can use `kubectl exec -it bash` to tunnel in and view the logs or debug the process.
+
+## Kubernetes concepts
+
+### Namespaces
+
+[Namespaces in Kubernetes](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) are a way to divide cluster resources between multiple users (via resource quota).
+It is similar to the queue concept in Yarn cluster. Flink on Kubernetes can use namespaces to launch Flink clusters.
+The namespace can be specified using the `-Dkubernetes.namespace=default` argument when starting a Flink cluster.
+
+[ResourceQuota](https://kubernetes.io/docs/concepts/policy/resource-quotas/) provides constraints that limit aggregate resource consumption per namespace.
+It can limit the quantity of objects that can be created in a namespace by type, as well as the total amount of compute resources that may be consumed by resources in that project.
+
+### RBAC
+
+Role-based access control ([RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)) is a method of regulating access to compute or network resources based on the roles of individual users within an enterprise.
+Users can configure RBAC roles and service accounts used by Flink JobManager to access the Kubernetes API server within the Kubernetes cluster.
+
+Every namespace has a default service account, however, the `default` service account may not have the permission to create or delete pods within the Kubernetes cluster.
+Users may need to update the permission of `default` service account or specify another service account that has the right role bound.
+
+{% highlight bash %}
+$ kubectl create clusterrolebinding flink-role-binding-default --clusterrole=edit --serviceaccount=default:default
+{% endhighlight %}
+
+If you do not want to use `default` service account, use the following command to create a new `flink` service account and set the role binding.
+Then use the config option `-Dkubernetes.jobmanager.service-account=flink` to make the JobManager pod using the `flink` service account to create and delete TaskManager pods.
+
+{% highlight bash %}
+$ kubectl create serviceaccount flink
+$ kubectl create clusterrolebinding flink-role-binding-flink --clusterrole=edit --serviceaccount=default:flink
+{% endhighlight %}
+
+Please reference the official Kubernetes documentation on [RBAC Authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) for more information.
+
+## Background / Internals
+
+This section briefly explains how Flink and Kubernetes interact.
+
+
+
+When creating a Flink Kubernetes session cluster, the Flink client will first connect to the Kubernetes ApiServer to submit the cluster description, including ConfigMap spec, Job Manager Service spec, Job Manager Deployment spec and Owner Reference.
+Kubernetes will then create the Flink master deployment, during which time the Kubelet will pull the image, prepare and mount the volume, and then execute the start command.
+After the master pod has launched, the Dispatcher and KubernetesResourceManager are available and the cluster is ready to accept one or more jobs.
+
+When users submit jobs through the Flink client, the job graph will be generated by the client and uploaded along with users jars to the Dispatcher.
+A JobMaster for that Job will be then be spawned.
+
+The JobMaster requests resources, known as slots, from the KubernetesResourceManager.
+If no slots are available, the resource manager will bring up TaskManager pods and registering them with the cluster.
+
+{% top %}
diff --git a/docs/ops/deployment/yarn_setup.md b/docs/ops/deployment/yarn_setup.md
index 544f7078342a75cdd048732915ac550af964bc60..2e04d45ebeba8cac110dc270946b18b4563dfbcf 100644
--- a/docs/ops/deployment/yarn_setup.md
+++ b/docs/ops/deployment/yarn_setup.md
@@ -2,7 +2,7 @@
title: "YARN Setup"
nav-title: YARN
nav-parent_id: deployment
-nav-pos: 2
+nav-pos: 4
---
+
+This section gives a detailed description of all components in Flink’s memory model of task executor.
+Check [memory configuration guide](mem_setup.html) for the basic memory setup.
+
+* toc
+{:toc}
+
+## Overview
+
+
+
+
+
+
+
+The following table lists all memory components, depicted above, and references Flink configuration options
+which affect the size of the respective components:
+
+| **Component** | **Configuration options** | **Description** |
+| :-------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Framework Heap Memory](#framework-memory) | [`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size) | JVM heap memory dedicated to Flink framework (advanced option) |
+| [Task Heap Memory](mem_setup.html#task-operator-heap-memory) | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | JVM heap memory dedicated to Flink application to run operators and user code |
+| [Managed memory](mem_setup.html#managed-memory) | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction) | Native memory managed by Flink, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backend |
+| [Framework Off-heap Memory](#framework-memory) | [`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink framework (advanced option) |
+| [Task Off-heap Memory](mem_setup.html#configure-off-heap-memory-direct-or-native) | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink application to run operators |
+| Network Memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) [`taskmanager.memory.network.fraction`](../config.html#taskmanager-memory-network-fraction) | Direct memory reserved for data record exchange between tasks (e.g. buffering for the transfer over the network), it is a [capped fractionated component](#capped-fractionated-components) of the [total Flink memory](mem_setup.html#configure-total-memory) |
+| [JVM metaspace](#jvm-parameters) | [`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size) | Metaspace size of the Flink JVM process |
+| JVM Overhead | [`taskmanager.memory.jvm-overhead.min`](../config.html#taskmanager-memory-jvm-overhead-min) [`taskmanager.memory.jvm-overhead.max`](../config.html#taskmanager-memory-jvm-overhead-max) [`taskmanager.memory.jvm-overhead.fraction`](../config.html#taskmanager-memory-jvm-overhead-fraction) | Native memory reserved for other JVM overhead: e.g. thread stacks, code cache, garbage collection space etc, it is a [capped fractionated component](#capped-fractionated-components) of the [total process memory](mem_setup.html#configure-total-memory) |
+{:.table-bordered}
+
+
+As you can see, the size of some memory components can be simply set by the respective option.
+Other components can be tuned using multiple options.
+
+## Framework Memory
+
+The *framework heap memory* and *framework off-heap memory* options are not supposed to be changed without a good reason.
+Adjust them only if you are sure that Flink needs more memory for some internal data structures or operations.
+It can be related to a particular deployment environment or job structure, like high parallelism.
+In addition, Flink dependencies, such as Hadoop may consume more direct or native memory in certain setups.
+
+Note Neither heap nor off-heap versions of framework and task memory are currently isolated within Flink.
+The separation of framework and task memory can be used in future releases for further optimizations.
+
+## Capped Fractionated Components
+
+This section describes the configuration details of the following options which can be a fraction of a certain
+[total memory](mem_setup.html#configure-total-memory):
+
+* *Network memory* can be a fraction of the *total Flink memory*
+* *JVM overhead* can be a fraction of the *total process memory*
+
+See also [detailed memory model](#overview).
+
+The size of those components always has to be between its maximum and minimum value, otherwise Flink startup will fail.
+The maximum and minimum values have defaults or can be explicitly set by corresponding configuration options.
+For example, if only the following memory options are set:
+- total Flink memory = 1000Mb,
+- network min = 64Mb,
+- network max = 128Mb,
+- network fraction = 0.1
+
+then the network memory will be 1000Mb x 0.1 = 100Mb which is within the range 64-128Mb.
+
+Notice if you configure the same maximum and minimum value it effectively means that its size is fixed to that value.
+
+If the component memory is not explicitly configured, then Flink will use the fraction to calculate the memory size
+based on the total memory. The calculated value is capped by its corresponding min/max options.
+For example, if only the following memory options are set:
+- total Flink memory = 1000Mb,
+- network min = 128Mb,
+- network max = 256Mb,
+- network fraction = 0.1
+
+then the network memory will be 128Mb because the size derived from fraction is 100Mb and it is less than the minimum.
+
+It can also happen that the fraction is ignored if the sizes of the total memory and its other components are defined.
+In this case, the network memory is the rest of the total memory. The derived value still has to be within its min/max
+range otherwise the configuration fails. For example, suppose only the following memory options are set:
+- total Flink memory = 1000Mb,
+- task heap = 100Mb,
+- network min = 64Mb,
+- network max = 256Mb,
+- network fraction = 0.1
+
+All other components of the total Flink memory have default values, including the default managed memory fraction.
+Then the network memory is not the fraction (1000Mb x 0.1 = 100Mb) but the rest of the total Flink memory
+which will either be within the range 64-256Mb or fail.
+
+## JVM Parameters
+
+Flink explicitly adds the following memory related JVM arguments while starting the task executor process,
+based on the configured or derived memory component sizes:
+
+| **JVM Arguments** | **Value** |
+| :---------------------------------------- | :----------------------------------------- |
+| *-Xmx* and *-Xms* | Framework + Task Heap Memory |
+| *-XX:MaxDirectMemorySize* | Framework + Task Off-Heap + Network Memory |
+| *-XX:MaxMetaspaceSize* | JVM Metaspace |
+{:.table-bordered}
+
+
+See also [detailed memory model](#overview).
+
+## Local Execution
+If you start Flink locally on your machine as a single java program without creating a cluster (e.g. from your IDE)
+then all components are ignored except for the following:
+
+| **Memory component** | **Relevant options** | **Default value for the local execution** |
+| :------------------------------------------- | :-------------------------------------------------------------------------------------------- | :---------------------------------------------------------------------------- |
+| Task heap | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | infinite |
+| Task off-heap | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | infinite |
+| Managed memory | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) | 128Mb |
+| Network memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) | 64Mb |
+{:.table-bordered}
+
+
+All of the components listed above can be but do not have to be explicitly configured for the local execution.
+If they are not configured they are set to their default values. [Task heap memory](mem_setup.html#task-operator-heap-memory) and
+*task off-heap memory* are considered to be infinite (*Long.MAX_VALUE* bytes) and [managed memory](mem_setup.html#managed-memory)
+has a default value of 128Mb only for the local execution mode.
+
+Note The task heap size is not related in any way to the real heap size in this case.
+It can become relevant for future optimizations coming with next releases. The actual JVM heap size of the started
+local process is not controlled by Flink and depends on how you start the process.
+If you want to control the JVM heap size you have to explicitly pass the corresponding JVM arguments, e.g. *-Xmx*, *-Xms*.
diff --git a/docs/ops/memory/mem_detail.zh.md b/docs/ops/memory/mem_detail.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..a5b476f1dd133472d3cbc8a3506df308163c7c68
--- /dev/null
+++ b/docs/ops/memory/mem_detail.zh.md
@@ -0,0 +1,149 @@
+---
+title: "Detailed Memory Model"
+nav-parent_id: ops_mem
+nav-pos: 2
+---
+
+
+This section gives a detailed description of all components in Flink’s memory model of task executor.
+Check [memory configuration guide](mem_setup.html) for the basic memory setup.
+
+* toc
+{:toc}
+
+## Overview
+
+
+
+
+
+
+
+The following table lists all memory components, depicted above, and references Flink configuration options
+which affect the size of the respective components:
+
+| **Component** | **Configuration options** | **Description** |
+| :-------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Framework Heap Memory](#framework-memory) | [`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size) | JVM heap memory dedicated to Flink framework (advanced option) |
+| [Task Heap Memory](mem_setup.html#task-operator-heap-memory) | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | JVM heap memory dedicated to Flink application to run operators and user code |
+| [Managed memory](mem_setup.html#managed-memory) | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction) | Native memory managed by Flink, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backend |
+| [Framework Off-heap Memory](#framework-memory) | [`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink framework (advanced option) |
+| [Task Off-heap Memory](mem_setup.html#configure-off-heap-memory-direct-or-native) | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink application to run operators |
+| Network Memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) [`taskmanager.memory.network.fraction`](../config.html#taskmanager-memory-network-fraction) | Direct memory reserved for data record exchange between tasks (e.g. buffering for the transfer over the network), it is a [capped fractionated component](#capped-fractionated-components) of the [total Flink memory](mem_setup.html#configure-total-memory) |
+| [JVM metaspace](#jvm-parameters) | [`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size) | Metaspace size of the Flink JVM process |
+| JVM Overhead | [`taskmanager.memory.jvm-overhead.min`](../config.html#taskmanager-memory-jvm-overhead-min) [`taskmanager.memory.jvm-overhead.max`](../config.html#taskmanager-memory-jvm-overhead-max) [`taskmanager.memory.jvm-overhead.fraction`](../config.html#taskmanager-memory-jvm-overhead-fraction) | Native memory reserved for other JVM overhead: e.g. thread stacks, code cache, garbage collection space etc, it is a [capped fractionated component](#capped-fractionated-components) of the [total process memory](mem_setup.html#configure-total-memory) |
+{:.table-bordered}
+
+
+As you can see, the size of some memory components can be simply set by the respective option.
+Other components can be tuned using multiple options.
+
+## Framework Memory
+
+The *framework heap memory* and *framework off-heap memory* options are not supposed to be changed without a good reason.
+Adjust them only if you are sure that Flink needs more memory for some internal data structures or operations.
+It can be related to a particular deployment environment or job structure, like high parallelism.
+In addition, Flink dependencies, such as Hadoop may consume more direct or native memory in certain setups.
+
+Note Neither heap nor off-heap versions of framework and task memory are currently isolated within Flink.
+The separation of framework and task memory can be used in future releases for further optimizations.
+
+## Capped Fractionated Components
+
+This section describes the configuration details of the following options which can be a fraction of a certain
+[total memory](mem_setup.html#configure-total-memory):
+
+* *Network memory* can be a fraction of the *total Flink memory*
+* *JVM overhead* can be a fraction of the *total process memory*
+
+See also [detailed memory model](#overview).
+
+The size of those components always has to be between its maximum and minimum value, otherwise Flink startup will fail.
+The maximum and minimum values have defaults or can be explicitly set by corresponding configuration options.
+For example, if only the following memory options are set:
+- total Flink memory = 1000Mb,
+- network min = 64Mb,
+- network max = 128Mb,
+- network fraction = 0.1
+
+then the network memory will be 1000Mb x 0.1 = 100Mb which is within the range 64-128Mb.
+
+Notice if you configure the same maximum and minimum value it effectively means that its size is fixed to that value.
+
+If the component memory is not explicitly configured, then Flink will use the fraction to calculate the memory size
+based on the total memory. The calculated value is capped by its corresponding min/max options.
+For example, if only the following memory options are set:
+- total Flink memory = 1000Mb,
+- network min = 128Mb,
+- network max = 256Mb,
+- network fraction = 0.1
+
+then the network memory will be 128Mb because the size derived from fraction is 100Mb and it is less than the minimum.
+
+It can also happen that the fraction is ignored if the sizes of the total memory and its other components are defined.
+In this case, the network memory is the rest of the total memory. The derived value still has to be within its min/max
+range otherwise the configuration fails. For example, suppose only the following memory options are set:
+- total Flink memory = 1000Mb,
+- task heap = 100Mb,
+- network min = 64Mb,
+- network max = 256Mb,
+- network fraction = 0.1
+
+All other components of the total Flink memory have default values, including the default managed memory fraction.
+Then the network memory is not the fraction (1000Mb x 0.1 = 100Mb) but the rest of the total Flink memory
+which will either be within the range 64-256Mb or fail.
+
+## JVM Parameters
+
+Flink explicitly adds the following memory related JVM arguments while starting the task executor process,
+based on the configured or derived memory component sizes:
+
+| **JVM Arguments** | **Value** |
+| :---------------------------------------- | :----------------------------------------- |
+| *-Xmx* and *-Xms* | Framework + Task Heap Memory |
+| *-XX:MaxDirectMemorySize* | Framework + Task Off-Heap + Network Memory |
+| *-XX:MaxMetaspaceSize* | JVM Metaspace |
+{:.table-bordered}
+
+
+See also [detailed memory model](#overview).
+
+## Local Execution
+If you start Flink locally on your machine as a single java program without creating a cluster (e.g. from your IDE)
+then all components are ignored except for the following:
+
+| **Memory component** | **Relevant options** | **Default value for the local execution** |
+| :------------------------------------------- | :-------------------------------------------------------------------------------------------- | :---------------------------------------------------------------------------- |
+| Task heap | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | infinite |
+| Task off-heap | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | infinite |
+| Managed memory | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) | 128Mb |
+| Network memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) | 64Mb |
+{:.table-bordered}
+
+
+All of the components listed above can be but do not have to be explicitly configured for the local execution.
+If they are not configured they are set to their default values. [Task heap memory](mem_setup.html#task-operator-heap-memory) and
+*task off-heap memory* are considered to be infinite (*Long.MAX_VALUE* bytes) and [managed memory](mem_setup.html#managed-memory)
+has a default value of 128Mb only for the local execution mode.
+
+Note The task heap size is not related in any way to the real heap size in this case.
+It can become relevant for future optimizations coming with next releases. The actual JVM heap size of the started
+local process is not controlled by Flink and depends on how you start the process.
+If you want to control the JVM heap size you have to explicitly pass the corresponding JVM arguments, e.g. *-Xmx*, *-Xms*.
diff --git a/docs/ops/memory/mem_migration.md b/docs/ops/memory/mem_migration.md
new file mode 100644
index 0000000000000000000000000000000000000000..dff7508b000e94e24d02d2d0f45f69b09b824bac
--- /dev/null
+++ b/docs/ops/memory/mem_migration.md
@@ -0,0 +1,236 @@
+---
+title: "Migration Guide"
+nav-parent_id: ops_mem
+nav-pos: 5
+---
+
+
+The [memory setup of task managers](mem_setup.html) has changed a lot with the 1.10 release. Many configuration options
+were removed or their semantics changed. This guide will help you to migrate the memory configuration from Flink
+[<= *1.9*](https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/mem_setup.html) to >= *1.10*.
+
+* toc
+{:toc}
+
+
+ Warning: It is important to review this guide because the legacy and new memory configuration can
+ result in different sizes of memory components. If you try to reuse your Flink configuration from older versions
+ before 1.10, it can result in changes to the behavior, performance or even configuration failures of your application.
+
+
+Note Before version *1.10*, Flink did not require that memory related options are set at all
+as they all had default values. The [new memory configuration](mem_setup.html#configure-total-memory) requires
+that at least one subset of the following options is configured explicitly, otherwise the configuration will fail:
+* [`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)
+* [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+* [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)
+
+The [default `flink-conf.yaml`](#default-configuration-in-flink-confyaml) shipped with Flink sets [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+to make the default memory configuration consistent.
+
+This [spreadsheet](https://docs.google.com/spreadsheets/d/1mJaMkMPfDJJ-w6nMXALYmTc4XxiV30P5U7DzgwLkSoE) can also help
+to evaluate and compare the results of the legacy and new memory computations.
+
+## Changes in Configuration Options
+
+This chapter shortly lists all changes to Flink's memory configuration options introduced with the *1.10* release.
+It also references other chapters for more details about migrating to the new configuration options.
+
+The following options are completely removed. If they are still used, they will be ignored.
+
+
+
+Although, the network memory configuration has not changed too much it is recommended to verify its configuration.
+It can change if other memory components have new sizes, e.g. the total memory which the network can be a fraction of.
+See also [new detailed memory model](mem_setup.html#detailed-memory-model).
+
+The container cut-off configuration options, [`containerized.heap-cutoff-ratio`](config.html#containerized-heap-cutoff-ratio)
+and [`containerized.heap-cutoff-min`](config.html#containerized-heap-cutoff-min), have no effect for task manager processes anymore
+but they still have the same semantics for the job manager process. See also [how to migrate container cut-off](#container-cut-off-memory).
+
+## Total Memory (Previously Heap Memory)
+
+The previous options which were responsible for the total memory used by Flink are `taskmanager.heap.size` or `taskmanager.heap.mb`.
+Despite their naming, they included not only JVM heap but also other off-heap memory components. The options have been deprecated.
+
+The Mesos integration also had a separate option with the same semantics: `mesos.resourcemanager.tasks.mem` which has also been removed.
+
+If the mentioned legacy options are used without specifying the corresponding new options,
+they will be directly translated into the following new options:
+* Total Flink memory ([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)) for standalone deployments
+* Total process memory ([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)) for containerized deployments (Yarn or Mesos)
+
+It is also recommended to use these new options instead of the legacy ones as they might be completely removed in the following releases.
+
+See also [how to configure total memory now](mem_setup.html#configure-total-memory).
+
+## JVM Heap Memory
+
+JVM heap memory previously consisted of the managed memory (if configured to be on-heap) and the rest
+which included any other usages of heap memory. This rest was always implicitly derived as the remaining part of the total memory,
+see also [how to migrate managed memory](#managed-memory).
+
+Now, if only *total Flink memory* or *total process memory* is configured, then the JVM heap is also derived as the rest of
+what is left after subtracting all other components from the total memory, see also [how to configure total memory](mem_setup.html#configure-total-memory).
+
+Additionally, you can now have more direct control over the JVM heap assigned to the operator tasks
+([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size)),
+see also [Task (Operator) Heap Memory](mem_setup.html#task-operator-heap-memory).
+The JVM heap memory is also used by the heap state backends ([MemoryStateBackend](../state/state_backends.html#the-memorystatebackend)
+or [FsStateBackend](../state/state_backends.html#the-fsstatebackend) if it is chosen for streaming jobs.
+
+A part of the JVM heap is now always reserved for Flink framework
+([`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size)).
+See also [Framework memory](mem_detail.html#framework-memory).
+
+## Managed Memory
+
+See also [how to configure managed memory now](mem_setup.html#managed-memory).
+
+### Explicit Size
+
+The previous option to configure managed memory size (`taskmanager.memory.size`) was renamed to
+[`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) and deprecated.
+It is recommended to use the new option because the legacy one can be removed in future releases.
+
+### Fraction
+
+If not set explicitly, the managed memory could be previously specified as a fraction (`taskmanager.memory.fraction`)
+of the total memory minus network memory and container cut-off (only for [Yarn](../deployment/yarn_setup.html) and
+[Mesos](../deployment/mesos.html) deployments). This option has been completely removed and will have no effect if still used.
+Please, use the new option [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction) instead.
+This new option will set the [managed memory](mem_setup.html#managed-memory) to the specified fraction of the
+[total Flink memory](mem_setup.html#configure-total-memory) if its size is not set explicitly by
+[`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size).
+
+### RocksDB state
+
+If the [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend) is chosen for a streaming job,
+its native memory consumption should now be accounted for in [managed memory](mem_setup.html#managed-memory).
+The RocksDB memory allocation is limited by the [managed memory](mem_setup.html#managed-memory) size.
+This should prevent the killing of containers on [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html).
+You can disable the RocksDB memory control by setting [state.backend.rocksdb.memory.managed](../config.html#state-backend-rocksdb-memory-managed)
+to `false`. See also [how to migrate container cut-off](#container-cut-off-memory).
+
+### Other changes
+
+Additionally, the following changes have been made:
+* The [managed memory](mem_setup.html#managed-memory) is always off-heap now. The configuration option `taskmanager.memory.off-heap` is removed and will have no effect anymore.
+* The [managed memory](mem_setup.html#managed-memory) now uses native memory which is not direct memory. It means that the managed memory is no longer accounted for in the JVM direct memory limit.
+* The [managed memory](mem_setup.html#managed-memory) is always lazily allocated now. The configuration option `taskmanager.memory.preallocate` is removed and will have no effect anymore.
+
+## Container Cut-Off Memory
+
+For containerized deployments, you could previously specify a cut-off memory. This memory could accommodate for unaccounted memory allocations.
+Dependencies which were not directly controlled by Flink were the main source of those allocations, e.g. RocksDB, internals of JVM, etc.
+This is no longer available and the related configuration options (`containerized.heap-cutoff-ratio` and `containerized.heap-cutoff-min`)
+will have no effect on the task manager process anymore. The new memory model introduced more specific memory components,
+described further, to address these concerns.
+
+In streaming jobs which use [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend), the RocksDB
+native memory consumption should be accounted for as a part of the [managed memory](mem_setup.html#managed-memory) now.
+The RocksDB memory allocation is also limited by the configured size of the [managed memory](mem_setup.html#managed-memory).
+See also [migrating managed memory](#managed-memory) and [how to configure managed memory now](mem_setup.html#managed-memory).
+
+The other direct or native off-heap memory consumers can now be addressed by the following new configuration options:
+* Task off-heap memory ([`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size))
+* Framework off-heap memory ([`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size))
+* JVM metaspace ([`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size))
+* JVM overhead (see also [detailed new memory model](mem_setup.html#detailed-memory-model))
+
+Note The job manager still has container cut-off memory configuration options.
+The mentioned configuration options remain valid for the job manager in the same way as before.
+
+## Default Configuration in flink-conf.yaml
+
+This section describes the changes of the default `flink-conf.yaml` shipped with Flink.
+
+The total memory (`taskmanager.heap.size`) is replaced by [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+in the default `flink-conf.yaml`. The value is also increased from 1024Mb to 1568Mb.
+See also [how to configure total memory now](mem_setup.html#configure-total-memory).
+
+
+ Warning: If you use the new default `flink-conf.yaml` it can result in different sizes of
+ the memory components and can lead to performance changes.
+
diff --git a/docs/ops/memory/mem_migration.zh.md b/docs/ops/memory/mem_migration.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..dff7508b000e94e24d02d2d0f45f69b09b824bac
--- /dev/null
+++ b/docs/ops/memory/mem_migration.zh.md
@@ -0,0 +1,236 @@
+---
+title: "Migration Guide"
+nav-parent_id: ops_mem
+nav-pos: 5
+---
+
+
+The [memory setup of task managers](mem_setup.html) has changed a lot with the 1.10 release. Many configuration options
+were removed or their semantics changed. This guide will help you to migrate the memory configuration from Flink
+[<= *1.9*](https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/mem_setup.html) to >= *1.10*.
+
+* toc
+{:toc}
+
+
+ Warning: It is important to review this guide because the legacy and new memory configuration can
+ result in different sizes of memory components. If you try to reuse your Flink configuration from older versions
+ before 1.10, it can result in changes to the behavior, performance or even configuration failures of your application.
+
+
+Note Before version *1.10*, Flink did not require that memory related options are set at all
+as they all had default values. The [new memory configuration](mem_setup.html#configure-total-memory) requires
+that at least one subset of the following options is configured explicitly, otherwise the configuration will fail:
+* [`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)
+* [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+* [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)
+
+The [default `flink-conf.yaml`](#default-configuration-in-flink-confyaml) shipped with Flink sets [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+to make the default memory configuration consistent.
+
+This [spreadsheet](https://docs.google.com/spreadsheets/d/1mJaMkMPfDJJ-w6nMXALYmTc4XxiV30P5U7DzgwLkSoE) can also help
+to evaluate and compare the results of the legacy and new memory computations.
+
+## Changes in Configuration Options
+
+This chapter shortly lists all changes to Flink's memory configuration options introduced with the *1.10* release.
+It also references other chapters for more details about migrating to the new configuration options.
+
+The following options are completely removed. If they are still used, they will be ignored.
+
+
+
+Although, the network memory configuration has not changed too much it is recommended to verify its configuration.
+It can change if other memory components have new sizes, e.g. the total memory which the network can be a fraction of.
+See also [new detailed memory model](mem_setup.html#detailed-memory-model).
+
+The container cut-off configuration options, [`containerized.heap-cutoff-ratio`](config.html#containerized-heap-cutoff-ratio)
+and [`containerized.heap-cutoff-min`](config.html#containerized-heap-cutoff-min), have no effect for task manager processes anymore
+but they still have the same semantics for the job manager process. See also [how to migrate container cut-off](#container-cut-off-memory).
+
+## Total Memory (Previously Heap Memory)
+
+The previous options which were responsible for the total memory used by Flink are `taskmanager.heap.size` or `taskmanager.heap.mb`.
+Despite their naming, they included not only JVM heap but also other off-heap memory components. The options have been deprecated.
+
+The Mesos integration also had a separate option with the same semantics: `mesos.resourcemanager.tasks.mem` which has also been removed.
+
+If the mentioned legacy options are used without specifying the corresponding new options,
+they will be directly translated into the following new options:
+* Total Flink memory ([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)) for standalone deployments
+* Total process memory ([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)) for containerized deployments (Yarn or Mesos)
+
+It is also recommended to use these new options instead of the legacy ones as they might be completely removed in the following releases.
+
+See also [how to configure total memory now](mem_setup.html#configure-total-memory).
+
+## JVM Heap Memory
+
+JVM heap memory previously consisted of the managed memory (if configured to be on-heap) and the rest
+which included any other usages of heap memory. This rest was always implicitly derived as the remaining part of the total memory,
+see also [how to migrate managed memory](#managed-memory).
+
+Now, if only *total Flink memory* or *total process memory* is configured, then the JVM heap is also derived as the rest of
+what is left after subtracting all other components from the total memory, see also [how to configure total memory](mem_setup.html#configure-total-memory).
+
+Additionally, you can now have more direct control over the JVM heap assigned to the operator tasks
+([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size)),
+see also [Task (Operator) Heap Memory](mem_setup.html#task-operator-heap-memory).
+The JVM heap memory is also used by the heap state backends ([MemoryStateBackend](../state/state_backends.html#the-memorystatebackend)
+or [FsStateBackend](../state/state_backends.html#the-fsstatebackend) if it is chosen for streaming jobs.
+
+A part of the JVM heap is now always reserved for Flink framework
+([`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size)).
+See also [Framework memory](mem_detail.html#framework-memory).
+
+## Managed Memory
+
+See also [how to configure managed memory now](mem_setup.html#managed-memory).
+
+### Explicit Size
+
+The previous option to configure managed memory size (`taskmanager.memory.size`) was renamed to
+[`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) and deprecated.
+It is recommended to use the new option because the legacy one can be removed in future releases.
+
+### Fraction
+
+If not set explicitly, the managed memory could be previously specified as a fraction (`taskmanager.memory.fraction`)
+of the total memory minus network memory and container cut-off (only for [Yarn](../deployment/yarn_setup.html) and
+[Mesos](../deployment/mesos.html) deployments). This option has been completely removed and will have no effect if still used.
+Please, use the new option [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction) instead.
+This new option will set the [managed memory](mem_setup.html#managed-memory) to the specified fraction of the
+[total Flink memory](mem_setup.html#configure-total-memory) if its size is not set explicitly by
+[`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size).
+
+### RocksDB state
+
+If the [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend) is chosen for a streaming job,
+its native memory consumption should now be accounted for in [managed memory](mem_setup.html#managed-memory).
+The RocksDB memory allocation is limited by the [managed memory](mem_setup.html#managed-memory) size.
+This should prevent the killing of containers on [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html).
+You can disable the RocksDB memory control by setting [state.backend.rocksdb.memory.managed](../config.html#state-backend-rocksdb-memory-managed)
+to `false`. See also [how to migrate container cut-off](#container-cut-off-memory).
+
+### Other changes
+
+Additionally, the following changes have been made:
+* The [managed memory](mem_setup.html#managed-memory) is always off-heap now. The configuration option `taskmanager.memory.off-heap` is removed and will have no effect anymore.
+* The [managed memory](mem_setup.html#managed-memory) now uses native memory which is not direct memory. It means that the managed memory is no longer accounted for in the JVM direct memory limit.
+* The [managed memory](mem_setup.html#managed-memory) is always lazily allocated now. The configuration option `taskmanager.memory.preallocate` is removed and will have no effect anymore.
+
+## Container Cut-Off Memory
+
+For containerized deployments, you could previously specify a cut-off memory. This memory could accommodate for unaccounted memory allocations.
+Dependencies which were not directly controlled by Flink were the main source of those allocations, e.g. RocksDB, internals of JVM, etc.
+This is no longer available and the related configuration options (`containerized.heap-cutoff-ratio` and `containerized.heap-cutoff-min`)
+will have no effect on the task manager process anymore. The new memory model introduced more specific memory components,
+described further, to address these concerns.
+
+In streaming jobs which use [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend), the RocksDB
+native memory consumption should be accounted for as a part of the [managed memory](mem_setup.html#managed-memory) now.
+The RocksDB memory allocation is also limited by the configured size of the [managed memory](mem_setup.html#managed-memory).
+See also [migrating managed memory](#managed-memory) and [how to configure managed memory now](mem_setup.html#managed-memory).
+
+The other direct or native off-heap memory consumers can now be addressed by the following new configuration options:
+* Task off-heap memory ([`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size))
+* Framework off-heap memory ([`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size))
+* JVM metaspace ([`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size))
+* JVM overhead (see also [detailed new memory model](mem_setup.html#detailed-memory-model))
+
+Note The job manager still has container cut-off memory configuration options.
+The mentioned configuration options remain valid for the job manager in the same way as before.
+
+## Default Configuration in flink-conf.yaml
+
+This section describes the changes of the default `flink-conf.yaml` shipped with Flink.
+
+The total memory (`taskmanager.heap.size`) is replaced by [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+in the default `flink-conf.yaml`. The value is also increased from 1024Mb to 1568Mb.
+See also [how to configure total memory now](mem_setup.html#configure-total-memory).
+
+
+ Warning: If you use the new default `flink-conf.yaml` it can result in different sizes of
+ the memory components and can lead to performance changes.
+
diff --git a/docs/ops/memory/mem_setup.md b/docs/ops/memory/mem_setup.md
new file mode 100644
index 0000000000000000000000000000000000000000..7e27ee2cf4dd3943d02ffb4ad1c9f57cfb108b07
--- /dev/null
+++ b/docs/ops/memory/mem_setup.md
@@ -0,0 +1,133 @@
+---
+title: "Set up Task Executor Memory"
+nav-parent_id: ops_mem
+nav-pos: 1
+---
+
+
+Apache Flink provides efficient workloads on top of the JVM by tightly controlling the memory usage of its various components.
+While the community strives to offer sensible defaults to all configurations, the full breadth of applications
+that users deploy on Flink means this isn't always possible. To provide the most production value to our users,
+Flink allows both high level and fine-grained tuning of memory allocation within clusters.
+
+* toc
+{:toc}
+
+The further described memory configuration is applicable starting with the release version *1.10*. If you upgrade Flink
+from earlier versions, check the [migration guide](mem_migration.html) because many changes were introduced with the *1.10* release.
+
+Note This memory setup guide is relevant only for task executors!
+Check [job manager related configuration options](../config.html#jobmanager-heap-size) for the memory setup of job manager.
+
+## Configure Total Memory
+
+The *total process memory* of Flink JVM processes consists of memory consumed by Flink application (*total Flink memory*)
+and by the JVM to run the process. The *total Flink memory* consumption includes usage of JVM heap,
+*managed memory* (managed by Flink) and other direct (or native) memory.
+
+
+
+
+
+
+If you run FIink locally (e.g. from your IDE) without creating a cluster, then only a subset of the memory configuration
+options are relevant, see also [local execution](mem_detail.html#local-execution) for more details.
+
+Otherwise, the simplest way to setup memory in Flink is to configure either of the two following options:
+* Total Flink memory ([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size))
+* Total process memory ([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size))
+
+The rest of the memory components will be adjusted automatically, based on default values or additionally configured options.
+[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components.
+
+Configuring *total Flink memory* is better suited for standalone deployments where you want to declare how much memory
+is given to Flink itself. The *total Flink memory* splits up into JVM heap, [managed memory size](#managed-memory)
+and *direct memory*.
+
+If you configure *total process memory* you declare how much memory in total should be assigned to the Flink *JVM process*.
+For the containerized deployments it corresponds to the size of the requested container, see also
+[how to configure memory for containers](mem_tuning.html#configure-memory-for-containers)
+([Kubernetes](../deployment/kubernetes.html), [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html)).
+
+Another way to setup the memory is to set [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory)
+([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)).
+This more fine-grained approach is described in more detail [here](#configure-heap-and-managed-memory).
+
+Note One of the three mentioned ways has to be used to configure Flink’s memory (except for local execution), or the Flink startup will fail.
+This means that one of the following option subsets, which do not have default values, have to be configured explicitly:
+* [`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)
+* [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+* [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)
+
+Note Explicitly configuring both *total process memory* and *total Flink memory* is not recommended.
+It may lead to deployment failures due to potential memory configuration conflicts. Additional configuration
+of other memory components also requires caution as it can produce further configuration conflicts.
+
+## Configure Heap and Managed Memory
+
+As mentioned before in [total memory description](#configure-total-memory), another way to setup memory in Flink is
+to specify explicitly both [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory).
+It gives more control over the available JVM heap to Flink’s tasks and its [managed memory](#managed-memory).
+
+The rest of the memory components will be adjusted automatically, based on default values or additionally configured options.
+[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components.
+
+Note If you have configured the task heap and managed memory explicitly, it is recommended to set neither
+*total process memory* nor *total Flink memory*. Otherwise, it may easily lead to memory configuration conflicts.
+
+### Task (Operator) Heap Memory
+
+If you want to guarantee that a certain amount of JVM heap is available for your user code, you can set the *task heap memory*
+explicitly ([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size)).
+It will be added to the JVM heap size and will be dedicated to Flink’s operators running the user code.
+
+### Managed Memory
+
+*Managed memory* is managed by Flink and is allocated as native memory (off-heap). The following workloads use *managed memory*:
+* Streaming jobs can use it for [RocksDB state backend](../state/state_backends.html#the-rocksdbstatebackend).
+* [Batch jobs](../../dev/batch) can use it for sorting, hash tables, caching of intermediate results.
+
+The size of *managed memory* can be
+* either configured explicitly via [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)
+* or computed as a fraction of *total Flink memory* via [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction).
+
+*Size* will override *fraction*, if both are set.
+If neither *size* nor *fraction* is explicitly configured, the [default fraction](../config.html#taskmanager-memory-managed-fraction) will be used.
+
+See also [how to configure memory for state backends](mem_tuning.html#configure-memory-for-state-backends) and [batch jobs](mem_tuning.html#configure-memory-for-batch-jobs).
+
+## Configure Off-Heap Memory (direct or native)
+
+The off-heap memory which is allocated by user code should be accounted for in *task off-heap memory*
+([`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size)).
+
+Note You can also adjust the [framework off-heap memory](mem_detail.html#framework-memory). This option is advanced
+and only recommended to be changed if you are sure that the Flink framework needs more memory.
+
+Flink includes the *framework off-heap memory* and *task off-heap memory* into the *direct memory* limit of the JVM,
+see also [JVM parameters](mem_detail.html#jvm-parameters).
+
+Note Although, native non-direct memory usage can be accounted for as a part of the
+*framework off-heap memory* or *task off-heap memory*, it will result in a higher JVM's *direct memory* limit in this case.
+
+Note The *network memory* is also part of JVM *direct memory* but it is managed by Flink and guaranteed
+to never exceed its configured size. Therefore, resizing the *network memory* will not help in this situation.
+
+See also [the detailed memory model](mem_detail.html#detailed-memory-model).
diff --git a/docs/ops/memory/mem_setup.zh.md b/docs/ops/memory/mem_setup.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..7e27ee2cf4dd3943d02ffb4ad1c9f57cfb108b07
--- /dev/null
+++ b/docs/ops/memory/mem_setup.zh.md
@@ -0,0 +1,133 @@
+---
+title: "Set up Task Executor Memory"
+nav-parent_id: ops_mem
+nav-pos: 1
+---
+
+
+Apache Flink provides efficient workloads on top of the JVM by tightly controlling the memory usage of its various components.
+While the community strives to offer sensible defaults to all configurations, the full breadth of applications
+that users deploy on Flink means this isn't always possible. To provide the most production value to our users,
+Flink allows both high level and fine-grained tuning of memory allocation within clusters.
+
+* toc
+{:toc}
+
+The further described memory configuration is applicable starting with the release version *1.10*. If you upgrade Flink
+from earlier versions, check the [migration guide](mem_migration.html) because many changes were introduced with the *1.10* release.
+
+Note This memory setup guide is relevant only for task executors!
+Check [job manager related configuration options](../config.html#jobmanager-heap-size) for the memory setup of job manager.
+
+## Configure Total Memory
+
+The *total process memory* of Flink JVM processes consists of memory consumed by Flink application (*total Flink memory*)
+and by the JVM to run the process. The *total Flink memory* consumption includes usage of JVM heap,
+*managed memory* (managed by Flink) and other direct (or native) memory.
+
+
+
+
+
+
+If you run FIink locally (e.g. from your IDE) without creating a cluster, then only a subset of the memory configuration
+options are relevant, see also [local execution](mem_detail.html#local-execution) for more details.
+
+Otherwise, the simplest way to setup memory in Flink is to configure either of the two following options:
+* Total Flink memory ([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size))
+* Total process memory ([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size))
+
+The rest of the memory components will be adjusted automatically, based on default values or additionally configured options.
+[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components.
+
+Configuring *total Flink memory* is better suited for standalone deployments where you want to declare how much memory
+is given to Flink itself. The *total Flink memory* splits up into JVM heap, [managed memory size](#managed-memory)
+and *direct memory*.
+
+If you configure *total process memory* you declare how much memory in total should be assigned to the Flink *JVM process*.
+For the containerized deployments it corresponds to the size of the requested container, see also
+[how to configure memory for containers](mem_tuning.html#configure-memory-for-containers)
+([Kubernetes](../deployment/kubernetes.html), [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html)).
+
+Another way to setup the memory is to set [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory)
+([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)).
+This more fine-grained approach is described in more detail [here](#configure-heap-and-managed-memory).
+
+Note One of the three mentioned ways has to be used to configure Flink’s memory (except for local execution), or the Flink startup will fail.
+This means that one of the following option subsets, which do not have default values, have to be configured explicitly:
+* [`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)
+* [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)
+* [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)
+
+Note Explicitly configuring both *total process memory* and *total Flink memory* is not recommended.
+It may lead to deployment failures due to potential memory configuration conflicts. Additional configuration
+of other memory components also requires caution as it can produce further configuration conflicts.
+
+## Configure Heap and Managed Memory
+
+As mentioned before in [total memory description](#configure-total-memory), another way to setup memory in Flink is
+to specify explicitly both [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory).
+It gives more control over the available JVM heap to Flink’s tasks and its [managed memory](#managed-memory).
+
+The rest of the memory components will be adjusted automatically, based on default values or additionally configured options.
+[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components.
+
+Note If you have configured the task heap and managed memory explicitly, it is recommended to set neither
+*total process memory* nor *total Flink memory*. Otherwise, it may easily lead to memory configuration conflicts.
+
+### Task (Operator) Heap Memory
+
+If you want to guarantee that a certain amount of JVM heap is available for your user code, you can set the *task heap memory*
+explicitly ([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size)).
+It will be added to the JVM heap size and will be dedicated to Flink’s operators running the user code.
+
+### Managed Memory
+
+*Managed memory* is managed by Flink and is allocated as native memory (off-heap). The following workloads use *managed memory*:
+* Streaming jobs can use it for [RocksDB state backend](../state/state_backends.html#the-rocksdbstatebackend).
+* [Batch jobs](../../dev/batch) can use it for sorting, hash tables, caching of intermediate results.
+
+The size of *managed memory* can be
+* either configured explicitly via [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)
+* or computed as a fraction of *total Flink memory* via [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction).
+
+*Size* will override *fraction*, if both are set.
+If neither *size* nor *fraction* is explicitly configured, the [default fraction](../config.html#taskmanager-memory-managed-fraction) will be used.
+
+See also [how to configure memory for state backends](mem_tuning.html#configure-memory-for-state-backends) and [batch jobs](mem_tuning.html#configure-memory-for-batch-jobs).
+
+## Configure Off-Heap Memory (direct or native)
+
+The off-heap memory which is allocated by user code should be accounted for in *task off-heap memory*
+([`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size)).
+
+Note You can also adjust the [framework off-heap memory](mem_detail.html#framework-memory). This option is advanced
+and only recommended to be changed if you are sure that the Flink framework needs more memory.
+
+Flink includes the *framework off-heap memory* and *task off-heap memory* into the *direct memory* limit of the JVM,
+see also [JVM parameters](mem_detail.html#jvm-parameters).
+
+Note Although, native non-direct memory usage can be accounted for as a part of the
+*framework off-heap memory* or *task off-heap memory*, it will result in a higher JVM's *direct memory* limit in this case.
+
+Note The *network memory* is also part of JVM *direct memory* but it is managed by Flink and guaranteed
+to never exceed its configured size. Therefore, resizing the *network memory* will not help in this situation.
+
+See also [the detailed memory model](mem_detail.html#detailed-memory-model).
diff --git a/docs/ops/memory/mem_trouble.md b/docs/ops/memory/mem_trouble.md
new file mode 100644
index 0000000000000000000000000000000000000000..cd6463bc1b88d073699e8fb38363d34b5e84d84f
--- /dev/null
+++ b/docs/ops/memory/mem_trouble.md
@@ -0,0 +1,74 @@
+---
+title: "Troubleshooting"
+nav-parent_id: ops_mem
+nav-pos: 4
+---
+
+
+* toc
+{:toc}
+
+## IllegalConfigurationException
+
+If you see an *IllegalConfigurationException* thrown from *TaskExecutorProcessUtils*, it usually indicates
+that there is either an invalid configuration value (e.g. negative memory size, fraction that is greater than 1, etc.)
+or configuration conflicts. Check the documentation chapters related to the [memory components](mem_setup.html#detailed-memory-model)
+mentioned in the exception message.
+
+## OutOfMemoryError: Java heap space
+
+The exception usually indicates that the JVM heap is too small. You can try to increase the JVM heap size
+by increasing [total memory](mem_setup.html#configure-total-memory) or [task heap memory](mem_setup.html#task-operator-heap-memory).
+
+Note You can also increase the [framework heap memory](mem_detail.html#framework-memory) but this option
+is advanced and should only be changed if you are sure that the Flink framework itself needs more memory.
+
+## OutOfMemoryError: Direct buffer memory
+
+The exception usually indicates that the JVM *direct memory* limit is too small or that there is a *direct memory leak*.
+Check whether user code or other external dependencies use the JVM *direct memory* and that it is properly accounted for.
+You can try to increase its limit by adjusting [direct off-heap memory](mem_setup.html#detailed-memory-model).
+See also [how to configure off-heap memory](mem_setup.html#configure-off-heap-memory-direct-or-native) and
+the [JVM arguments](mem_detail.html#jvm-parameters) which Flink sets.
+
+## OutOfMemoryError: Metaspace
+
+The exception usually indicates that [JVM metaspace limit](mem_detail.html#jvm-parameters) is configured too small.
+You can try to increase the [JVM metaspace option](../config.html#taskmanager-memory-jvm-metaspace-size).
+
+## IOException: Insufficient number of network buffers
+
+The exception usually indicates that the size of the configured [network memory](mem_setup.html#detailed-memory-model)
+is not big enough. You can try to increase the *network memory* by adjusting the following options:
+* [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min)
+* [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max)
+* [`taskmanager.memory.network.fraction`](../config.html#taskmanager-memory-network-fraction)
+
+## Container Memory Exceeded
+
+If a task executor container tries to allocate memory beyond its requested size (Yarn, Mesos or Kubernetes),
+this usually indicates that Flink has not reserved enough native memory. You can observe this either by using an external
+monitoring system or from the error messages when a container gets killed by the deployment environment.
+
+If [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend) is used and the memory controlling is disabled,
+you can try to increase the [managed memory](mem_setup.html#managed-memory).
+
+Alternatively, you can increase the [JVM overhead](mem_setup.html#detailed-memory-model).
+See also [how to configure memory for containers](mem_tuning.html#configure-memory-for-containers).
diff --git a/docs/ops/memory/mem_trouble.zh.md b/docs/ops/memory/mem_trouble.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..cd6463bc1b88d073699e8fb38363d34b5e84d84f
--- /dev/null
+++ b/docs/ops/memory/mem_trouble.zh.md
@@ -0,0 +1,74 @@
+---
+title: "Troubleshooting"
+nav-parent_id: ops_mem
+nav-pos: 4
+---
+
+
+* toc
+{:toc}
+
+## IllegalConfigurationException
+
+If you see an *IllegalConfigurationException* thrown from *TaskExecutorProcessUtils*, it usually indicates
+that there is either an invalid configuration value (e.g. negative memory size, fraction that is greater than 1, etc.)
+or configuration conflicts. Check the documentation chapters related to the [memory components](mem_setup.html#detailed-memory-model)
+mentioned in the exception message.
+
+## OutOfMemoryError: Java heap space
+
+The exception usually indicates that the JVM heap is too small. You can try to increase the JVM heap size
+by increasing [total memory](mem_setup.html#configure-total-memory) or [task heap memory](mem_setup.html#task-operator-heap-memory).
+
+Note You can also increase the [framework heap memory](mem_detail.html#framework-memory) but this option
+is advanced and should only be changed if you are sure that the Flink framework itself needs more memory.
+
+## OutOfMemoryError: Direct buffer memory
+
+The exception usually indicates that the JVM *direct memory* limit is too small or that there is a *direct memory leak*.
+Check whether user code or other external dependencies use the JVM *direct memory* and that it is properly accounted for.
+You can try to increase its limit by adjusting [direct off-heap memory](mem_setup.html#detailed-memory-model).
+See also [how to configure off-heap memory](mem_setup.html#configure-off-heap-memory-direct-or-native) and
+the [JVM arguments](mem_detail.html#jvm-parameters) which Flink sets.
+
+## OutOfMemoryError: Metaspace
+
+The exception usually indicates that [JVM metaspace limit](mem_detail.html#jvm-parameters) is configured too small.
+You can try to increase the [JVM metaspace option](../config.html#taskmanager-memory-jvm-metaspace-size).
+
+## IOException: Insufficient number of network buffers
+
+The exception usually indicates that the size of the configured [network memory](mem_setup.html#detailed-memory-model)
+is not big enough. You can try to increase the *network memory* by adjusting the following options:
+* [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min)
+* [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max)
+* [`taskmanager.memory.network.fraction`](../config.html#taskmanager-memory-network-fraction)
+
+## Container Memory Exceeded
+
+If a task executor container tries to allocate memory beyond its requested size (Yarn, Mesos or Kubernetes),
+this usually indicates that Flink has not reserved enough native memory. You can observe this either by using an external
+monitoring system or from the error messages when a container gets killed by the deployment environment.
+
+If [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend) is used and the memory controlling is disabled,
+you can try to increase the [managed memory](mem_setup.html#managed-memory).
+
+Alternatively, you can increase the [JVM overhead](mem_setup.html#detailed-memory-model).
+See also [how to configure memory for containers](mem_tuning.html#configure-memory-for-containers).
diff --git a/docs/ops/memory/mem_tuning.md b/docs/ops/memory/mem_tuning.md
new file mode 100644
index 0000000000000000000000000000000000000000..9d83d00d8dc772ba1ac2c3fb1a4b2d3dadf6fa7f
--- /dev/null
+++ b/docs/ops/memory/mem_tuning.md
@@ -0,0 +1,87 @@
+---
+title: "Memory tuning guide"
+nav-parent_id: ops_mem
+nav-pos: 3
+---
+
+
+In addition to the [main memory setup guide](mem_setup.html), this section explains how to setup memory of task executors
+depending on the use case and which options are important in which case.
+
+* toc
+{:toc}
+
+## Configure memory for standalone deployment
+
+It is recommended to configure [total Flink memory](mem_setup.html#configure-total-memory)
+([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)) or its [components](mem_setup.html#detailed-memory-model)
+for [standalone deployment](../deployment/cluster_setup.html) where you want to declare how much memory is given to Flink itself.
+Additionally, you can adjust *JVM metaspace* if it causes [problems](mem_trouble.html#outofmemoryerror-metaspace).
+
+The *total Process memory* is not relevant because *JVM overhead* is not controlled by Flink or deployment environment,
+only physical resources of the executing machine matter in this case.
+
+## Configure memory for containers
+
+It is recommended to configure [total process memory](mem_setup.html#configure-total-memory)
+([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)) for the containerized deployments
+([Kubernetes](../deployment/kubernetes.html), [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html)).
+It declares how much memory in total should be assigned to the Flink *JVM process* and corresponds to the size of the requested container.
+
+Note If you configure the *total Flink memory* Flink will implicitly add JVM memory components
+to derive the *total process memory* and request a container with the memory of that derived size,
+see also [detailed Memory Model](mem_setup.html#detailed-memory-model).
+
+
+ Warning: If Flink or user code allocates unmanaged off-heap (native) memory beyond the container size
+ the job can fail because the deployment environment can kill the offending containers.
+
+See also description of [container memory exceeded](mem_trouble.html#container-memory-exceeded) failure.
+
+## Configure memory for state backends
+
+When deploying a Flink streaming application, the type of [state backend](../state/state_backends.html) used
+will dictate the optimal memory configurations of your cluster.
+
+### Heap state backend
+
+When running a stateless job or using a heap state backend ([MemoryStateBackend](../state/state_backends.html#the-memorystatebackend)
+or [FsStateBackend](../state/state_backends.html#the-fsstatebackend), set [managed memory](mem_setup.html#managed-memory) to zero.
+This will ensure that the maximum amount of memory is allocated for user code on the JVM.
+
+### RocksDB state backend
+
+The [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend) uses native memory. By default,
+RocksDB is setup to limit native memory allocation to the size of the [managed memory](mem_setup.html#managed-memory).
+Therefore, it is important to reserve enough *managed memory* for your state use case. If you disable the default RocksDB memory control,
+task executors can be killed in containerized deployments if RocksDB allocates memory above the limit of the requested container size
+(the [total process memory](mem_setup.html#configure-total-memory)).
+See also [how to tune RocksDB memory](../state/large_state_tuning.html#tuning-rocksdb-memory)
+and [state.backend.rocksdb.memory.managed](../config.html#state-backend-rocksdb-memory-managed).
+
+## Configure memory for batch jobs
+
+Flink's batch operators leverage [managed memory](../memory/mem_setup.html#managed-memory) to run more efficiently.
+In doing so, some operations can be performed directly on raw data without having to be deserialized into Java objects.
+This means that [managed memory](../memory/mem_setup.html#managed-memory) configurations have practical effects
+on the performance of your applications. Flink will attempt to allocate and use as much [managed memory](../memory/mem_setup.html#managed-memory)
+as configured for batch jobs but not go beyond its limits. This prevents `OutOfMemoryError`'s because Flink knows precisely
+how much memory it has to leverage. If the [managed memory](../memory/mem_setup.html#managed-memory) is not sufficient,
+Flink will gracefully spill to disk.
diff --git a/docs/ops/memory/mem_tuning.zh.md b/docs/ops/memory/mem_tuning.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..9d83d00d8dc772ba1ac2c3fb1a4b2d3dadf6fa7f
--- /dev/null
+++ b/docs/ops/memory/mem_tuning.zh.md
@@ -0,0 +1,87 @@
+---
+title: "Memory tuning guide"
+nav-parent_id: ops_mem
+nav-pos: 3
+---
+
+
+In addition to the [main memory setup guide](mem_setup.html), this section explains how to setup memory of task executors
+depending on the use case and which options are important in which case.
+
+* toc
+{:toc}
+
+## Configure memory for standalone deployment
+
+It is recommended to configure [total Flink memory](mem_setup.html#configure-total-memory)
+([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)) or its [components](mem_setup.html#detailed-memory-model)
+for [standalone deployment](../deployment/cluster_setup.html) where you want to declare how much memory is given to Flink itself.
+Additionally, you can adjust *JVM metaspace* if it causes [problems](mem_trouble.html#outofmemoryerror-metaspace).
+
+The *total Process memory* is not relevant because *JVM overhead* is not controlled by Flink or deployment environment,
+only physical resources of the executing machine matter in this case.
+
+## Configure memory for containers
+
+It is recommended to configure [total process memory](mem_setup.html#configure-total-memory)
+([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)) for the containerized deployments
+([Kubernetes](../deployment/kubernetes.html), [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html)).
+It declares how much memory in total should be assigned to the Flink *JVM process* and corresponds to the size of the requested container.
+
+Note If you configure the *total Flink memory* Flink will implicitly add JVM memory components
+to derive the *total process memory* and request a container with the memory of that derived size,
+see also [detailed Memory Model](mem_setup.html#detailed-memory-model).
+
+
+ Warning: If Flink or user code allocates unmanaged off-heap (native) memory beyond the container size
+ the job can fail because the deployment environment can kill the offending containers.
+
+See also description of [container memory exceeded](mem_trouble.html#container-memory-exceeded) failure.
+
+## Configure memory for state backends
+
+When deploying a Flink streaming application, the type of [state backend](../state/state_backends.html) used
+will dictate the optimal memory configurations of your cluster.
+
+### Heap state backend
+
+When running a stateless job or using a heap state backend ([MemoryStateBackend](../state/state_backends.html#the-memorystatebackend)
+or [FsStateBackend](../state/state_backends.html#the-fsstatebackend), set [managed memory](mem_setup.html#managed-memory) to zero.
+This will ensure that the maximum amount of memory is allocated for user code on the JVM.
+
+### RocksDB state backend
+
+The [RocksDBStateBackend](../state/state_backends.html#the-rocksdbstatebackend) uses native memory. By default,
+RocksDB is setup to limit native memory allocation to the size of the [managed memory](mem_setup.html#managed-memory).
+Therefore, it is important to reserve enough *managed memory* for your state use case. If you disable the default RocksDB memory control,
+task executors can be killed in containerized deployments if RocksDB allocates memory above the limit of the requested container size
+(the [total process memory](mem_setup.html#configure-total-memory)).
+See also [how to tune RocksDB memory](../state/large_state_tuning.html#tuning-rocksdb-memory)
+and [state.backend.rocksdb.memory.managed](../config.html#state-backend-rocksdb-memory-managed).
+
+## Configure memory for batch jobs
+
+Flink's batch operators leverage [managed memory](../memory/mem_setup.html#managed-memory) to run more efficiently.
+In doing so, some operations can be performed directly on raw data without having to be deserialized into Java objects.
+This means that [managed memory](../memory/mem_setup.html#managed-memory) configurations have practical effects
+on the performance of your applications. Flink will attempt to allocate and use as much [managed memory](../memory/mem_setup.html#managed-memory)
+as configured for batch jobs but not go beyond its limits. This prevents `OutOfMemoryError`'s because Flink knows precisely
+how much memory it has to leverage. If the [managed memory](../memory/mem_setup.html#managed-memory) is not sufficient,
+Flink will gracefully spill to disk.
diff --git a/docs/ops/plugins.md b/docs/ops/plugins.md
new file mode 100644
index 0000000000000000000000000000000000000000..759d5dcf35fcce9c32c21f803d14b147440bc873
--- /dev/null
+++ b/docs/ops/plugins.md
@@ -0,0 +1,117 @@
+---
+title: "Plugins"
+nav-id: plugins
+nav-parent_id: ops
+nav-pos: 14
+---
+
+
+Plugins facilitate a strict separation of code through restricted classloaders. Plugins cannot
+access classes from other plugins or from Flink that have not been specifically whitelisted. This
+strict isolation allows plugins to contain conflicting versions of the same library without the need
+to relocate classes or to converge to common versions. Currently, only file systems are pluggable
+but in the future, connectors, formats, and even user code should also be pluggable.
+
+* This will be replaced by the TOC
+{:toc}
+
+## Isolation and plugin structure
+
+Plugins reside in their own folders and can consist of several jars. The names of the plugin folders
+are arbitrary.
+
+```
+flink-dist
+├── conf
+├── lib
+...
+└── plugins
+ ├── s3
+ │ ├── aws-credential-provider.jar
+ │ └── flink-s3-fs-hadoop.jar
+ └── azure
+ └── flink-azure-fs-hadoop.jar
+```
+
+Each plugin is loaded through its own classloader and completely isolated from any other plugin.
+Hence, the `flink-s3-fs-hadoop` and `flink-azure-fs-hadoop` can depend on different conflicting
+library versions. There is no need to relocate any class during the creation of fat jars (shading).
+
+Plugins may access certain whitelisted packages from Flink's `lib/` folder. In particular, all
+necessary service provider interfaces (SPI) are loaded through the system classloader, so that no
+two versions of `org.apache.flink.core.fs.FileSystem` exist at any given time, even if users
+accidentally bundle it in their fat jar. This singleton class requirement is strictly necessary so
+that the Flink runtime has an entry point into the plugin. Service classes are discovered through
+the `java.util.ServiceLoader`, so make sure to retain the service definitions in `META-INF/services`
+during shading.
+
+Note *Currently, more Flink core classes are still
+accessible from plugins as we flesh out the SPI system.*
+
+Furthermore, the most common logger frameworks are whitelisted, such that logging is uniformly
+possible across Flink core, plugins, and user code.
+
+## File Systems
+
+All [file systems](filesystems/index) **except MapR** are pluggable. That means they can and should
+be used as plugins. To use a pluggable file system, copy the corresponding JAR file from the `opt`
+directory to a directory under `plugins` directory of your Flink distribution before starting Flink,
+e.g.
+
+{% highlight bash %}
+mkdir ./plugins/s3-fs-hadoop
+cp ./opt/flink-s3-fs-hadoop-{{ site.version }}.jar ./plugins/s3-fs-hadoop/
+{% endhighlight %}
+
+Warning **The s3 file systems (`flink-s3-fs-presto` and
+`flink-s3-fs-hadoop`) can only be used as plugins as we already removed the relocations.** Placing
+them in libs/ will result in system failures.
+
+Attention **Because of the strict [isolation]
+(#isolation-and-plugin-structure), file systems do not have access to credential providers in lib/
+anymore.** Please add any needed providers to the respective plugin folder.
+
+
+
+
+
+{% top %}
diff --git a/docs/ops/plugins.zh.md b/docs/ops/plugins.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..759d5dcf35fcce9c32c21f803d14b147440bc873
--- /dev/null
+++ b/docs/ops/plugins.zh.md
@@ -0,0 +1,117 @@
+---
+title: "Plugins"
+nav-id: plugins
+nav-parent_id: ops
+nav-pos: 14
+---
+
+
+Plugins facilitate a strict separation of code through restricted classloaders. Plugins cannot
+access classes from other plugins or from Flink that have not been specifically whitelisted. This
+strict isolation allows plugins to contain conflicting versions of the same library without the need
+to relocate classes or to converge to common versions. Currently, only file systems are pluggable
+but in the future, connectors, formats, and even user code should also be pluggable.
+
+* This will be replaced by the TOC
+{:toc}
+
+## Isolation and plugin structure
+
+Plugins reside in their own folders and can consist of several jars. The names of the plugin folders
+are arbitrary.
+
+```
+flink-dist
+├── conf
+├── lib
+...
+└── plugins
+ ├── s3
+ │ ├── aws-credential-provider.jar
+ │ └── flink-s3-fs-hadoop.jar
+ └── azure
+ └── flink-azure-fs-hadoop.jar
+```
+
+Each plugin is loaded through its own classloader and completely isolated from any other plugin.
+Hence, the `flink-s3-fs-hadoop` and `flink-azure-fs-hadoop` can depend on different conflicting
+library versions. There is no need to relocate any class during the creation of fat jars (shading).
+
+Plugins may access certain whitelisted packages from Flink's `lib/` folder. In particular, all
+necessary service provider interfaces (SPI) are loaded through the system classloader, so that no
+two versions of `org.apache.flink.core.fs.FileSystem` exist at any given time, even if users
+accidentally bundle it in their fat jar. This singleton class requirement is strictly necessary so
+that the Flink runtime has an entry point into the plugin. Service classes are discovered through
+the `java.util.ServiceLoader`, so make sure to retain the service definitions in `META-INF/services`
+during shading.
+
+Note *Currently, more Flink core classes are still
+accessible from plugins as we flesh out the SPI system.*
+
+Furthermore, the most common logger frameworks are whitelisted, such that logging is uniformly
+possible across Flink core, plugins, and user code.
+
+## File Systems
+
+All [file systems](filesystems/index) **except MapR** are pluggable. That means they can and should
+be used as plugins. To use a pluggable file system, copy the corresponding JAR file from the `opt`
+directory to a directory under `plugins` directory of your Flink distribution before starting Flink,
+e.g.
+
+{% highlight bash %}
+mkdir ./plugins/s3-fs-hadoop
+cp ./opt/flink-s3-fs-hadoop-{{ site.version }}.jar ./plugins/s3-fs-hadoop/
+{% endhighlight %}
+
+Warning **The s3 file systems (`flink-s3-fs-presto` and
+`flink-s3-fs-hadoop`) can only be used as plugins as we already removed the relocations.** Placing
+them in libs/ will result in system failures.
+
+Attention **Because of the strict [isolation]
+(#isolation-and-plugin-structure), file systems do not have access to credential providers in lib/
+anymore.** Please add any needed providers to the respective plugin folder.
+
+
+
+
+
+{% top %}
diff --git a/docs/ops/production_ready.md b/docs/ops/production_ready.md
index ef971730b706c8ca701fedb5815835d084e7fa7b..dcd691329f7cbaed67bfa2cd6d487672cc68e1dd 100644
--- a/docs/ops/production_ready.md
+++ b/docs/ops/production_ready.md
@@ -1,7 +1,7 @@
---
title: "Production Readiness Checklist"
nav-parent_id: ops
-nav-pos: 5
+nav-pos: 6
---
diff --git a/docs/getting-started/tutorials/setup_instructions.md b/docs/redirects/gce_setup.md
similarity index 86%
rename from docs/getting-started/tutorials/setup_instructions.md
rename to docs/redirects/gce_setup.md
index 8fbf8f4959465f4aa06cc95b824f93f395f7a12e..cfdde179da6d7d82a2b7a475984a1560e4df09e0 100644
--- a/docs/getting-started/tutorials/setup_instructions.md
+++ b/docs/redirects/gce_setup.md
@@ -1,9 +1,8 @@
---
-title: "Setup Tutorials"
-nav-id: setuptutorials
-nav-title: 'Setup Tutorials'
-nav-parent_id: tutorials
-nav-pos: 20
+title: "Google Compute Engine Setup"
+layout: redirect
+redirect: /index.html
+permalink: /ops/deployment/gce_setup.html
---
diff --git a/docs/getting-started/tutorials/api_tutorials.md b/docs/redirects/mapr.md
similarity index 87%
rename from docs/getting-started/tutorials/api_tutorials.md
rename to docs/redirects/mapr.md
index 2592e7b67f6e929289cee5f9092cb55d0946b971..f72a9697e1b32efc143f6117fbe3d6f3c4412dea 100644
--- a/docs/getting-started/tutorials/api_tutorials.md
+++ b/docs/redirects/mapr.md
@@ -1,9 +1,8 @@
---
-title: "API Tutorials"
-nav-id: apitutorials
-nav-title: 'API Tutorials'
-nav-parent_id: tutorials
-nav-pos: 10
+title: "MapR Setup"
+layout: redirect
+redirect: /index.html
+permalink: /ops/deployment/mapr_setup.html
---
diff --git a/docs/redirects/projectsetup_dependencies.zh.md b/docs/redirects/projectsetup_dependencies.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..693bfa9e17ff394623701d89235c735feb16f14c
--- /dev/null
+++ b/docs/redirects/projectsetup_dependencies.zh.md
@@ -0,0 +1,24 @@
+---
+title: "配置依赖、连接器、类库"
+layout: redirect
+redirect: /getting-started/project-setup/dependencies.html
+permalink: /dev/projectsetup/dependencies.html
+---
+
diff --git a/docs/getting-started/tutorials/index.md b/docs/redirects/projectsetup_java_api_quickstart.md
similarity index 81%
rename from docs/getting-started/tutorials/index.md
rename to docs/redirects/projectsetup_java_api_quickstart.md
index 55ff249bd32355e1d40ff63589348e8989a086da..386f2864f138a3d581b940cf2d1abf0200349f21 100644
--- a/docs/getting-started/tutorials/index.md
+++ b/docs/redirects/projectsetup_java_api_quickstart.md
@@ -1,9 +1,8 @@
---
-title: "Tutorials"
-nav-id: tutorials
-nav-title: ' Tutorials'
-nav-parent_id: getting-started
-nav-pos: 30
+title: "Project Template for Java"
+layout: redirect
+redirect: /getting-started/project-setup/java_api_quickstart.html
+permalink: /dev/projectsetup/java_api_quickstart.html
---
diff --git a/docs/redirects/projectsetup_scala_api_quickstart.md b/docs/redirects/projectsetup_scala_api_quickstart.md
new file mode 100644
index 0000000000000000000000000000000000000000..d3137766c85a31f6bf889d753be6b4b1ecfac538
--- /dev/null
+++ b/docs/redirects/projectsetup_scala_api_quickstart.md
@@ -0,0 +1,24 @@
+---
+title: "Project Template for Scala"
+layout: redirect
+redirect: /getting-started/project-setup/scala_api_quickstart.html
+permalink: /dev/projectsetup/scala_api_quickstart.html
+---
+
diff --git a/docs/redirects/projectsetup_scala_api_quickstart.zh.md b/docs/redirects/projectsetup_scala_api_quickstart.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..3a0d01616f7781f59805127cf56227cae5c18a0d
--- /dev/null
+++ b/docs/redirects/projectsetup_scala_api_quickstart.zh.md
@@ -0,0 +1,24 @@
+---
+title: "Scala 项目模板"
+layout: redirect
+redirect: /getting-started/project-setup/scala_api_quickstart.html
+permalink: /dev/projectsetup/scala_api_quickstart.html
+---
+
diff --git a/docs/redirects/python_table_tutorial.md b/docs/redirects/python_table_tutorial.md
new file mode 100644
index 0000000000000000000000000000000000000000..e29e6b20c5063e9a1cbf09b9be472d51d93ffe0e
--- /dev/null
+++ b/docs/redirects/python_table_tutorial.md
@@ -0,0 +1,24 @@
+---
+title: "Python API Tutoria"
+layout: redirect
+redirect: /getting-started/walkthroughs/python_table_api.html
+permalink: /tutorials/python_table_api.html
+---
+
diff --git a/docs/redirects/setup_quickstart.md b/docs/redirects/setup_quickstart.md
index 70c5f1b475045fa2a6f1d9255614121de1a7cda8..90e2cdceecde3e148c8a8eb7ee5486423d58b17a 100644
--- a/docs/redirects/setup_quickstart.md
+++ b/docs/redirects/setup_quickstart.md
@@ -1,7 +1,7 @@
---
title: "Local Setup Tutorial"
layout: redirect
-redirect: /getting-started/tutorials/local_setup.html
+redirect: /ops/deployment/local.html
permalink: /quickstart/setup_quickstart.html
---
diff --git a/docs/redirects/windows_local_setup.md b/docs/redirects/windows_local_setup.md
new file mode 100644
index 0000000000000000000000000000000000000000..f65c3ebc52b1b99951f0c1f0ec49c24f17af2c81
--- /dev/null
+++ b/docs/redirects/windows_local_setup.md
@@ -0,0 +1,24 @@
+---
+title: "Running Flink on Windows"
+layout: redirect
+redirect: /ops/deployment/local.html
+permalink: /getting-started/tutorials/flink_on_windows.html
+---
+
diff --git a/docs/release-notes/flink-1.10.md b/docs/release-notes/flink-1.10.md
new file mode 100644
index 0000000000000000000000000000000000000000..0e228ec0f500cc51bdf32704042a5900f27707eb
--- /dev/null
+++ b/docs/release-notes/flink-1.10.md
@@ -0,0 +1,466 @@
+---
+title: "Release Notes - Flink 1.10"
+---
+
+
+
+These release notes discuss important aspects, such as configuration, behavior,
+or dependencies, that changed between Flink 1.9 and Flink 1.10. Please read
+these notes carefully if you are planning to upgrade your Flink version to 1.10.
+
+* This will be replaced by the TOC
+{:toc}
+
+
+### Clusters & Deployment
+#### FileSystems should be loaded via Plugin Architecture ([FLINK-11956](https://issues.apache.org/jira/browse/FLINK-11956))
+s3-hadoop and s3-presto filesystems do no longer use class relocations and need
+to be loaded through [plugins]({{ site.baseurl }}/ops/filesystems/#pluggable-file-systems)
+but now seamlessly integrate with all credential providers. Other filesystems
+are strongly recommended to be only used as plugins as we will continue to
+remove relocations.
+
+#### Flink Client respects Classloading Policy ([FLINK-13749](https://issues.apache.org/jira/browse/FLINK-13749))
+The Flink client now also respects the configured classloading policy, i.e.,
+`parent-first` or `child-first` classloading. Previously, only cluster
+components such as the job manager or task manager supported this setting.
+This does mean that users might get different behaviour in their programs, in
+which case they should configure the classloading policy explicitly to use
+`parent-first` classloading, which was the previous (hard-coded) behaviour.
+
+#### Enable spreading out Tasks evenly across all TaskManagers ([FLINK-12122](https://issues.apache.org/jira/browse/FLINK-12122))
+When
+[FLIP-6](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077)
+was rolled out with Flink 1.5.0, we changed how slots are allocated from
+TaskManagers (TMs). Instead of evenly allocating the slots from all registered
+TMs, we had the tendency to exhaust a TM before using another one. To use a
+scheduling strategy that is more similar to the pre-FLIP-6 behaviour, where
+Flink tries to spread out the workload across all currently available TMs, one
+can set `cluster.evenly-spread-out-slots: true` in the `flink-conf.yaml`.
+
+#### Directory Structure Change for highly available Artifacts ([FLINK-13633](https://issues.apache.org/jira/browse/FLINK-13633))
+All highly available artifacts stored by Flink will now be stored under
+`HA_STORAGE_DIR/HA_CLUSTER_ID` with `HA_STORAGE_DIR` configured by
+`high-availability.storageDir` and `HA_CLUSTER_ID` configured by
+`high-availability.cluster-id`.
+
+#### Resources and JARs shipped via --yarnship will be ordered in the Classpath ([FLINK-13127](https://issues.apache.org/jira/browse/FLINK-13127))
+When using the `--yarnship` command line option, resource directories and jar
+files will be added to the classpath in lexicographical order with resources
+directories appearing first.
+
+#### Removal of --yn/--yarncontainer Command Line Options ([FLINK-12362](https://issues.apache.org/jira/browse/FLINK-12362))
+The Flink CLI no longer supports the deprecated command line options
+`-yn/--yarncontainer`, which were used to specify the number of containers to
+start on YARN. This option has been deprecated since the introduction of
+[FLIP-6](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077).
+All Flink users are advised to remove this command line option.
+
+#### Removal of --yst/--yarnstreaming Command Line Options ([FLINK-14957](https://issues.apache.org/jira/browse/FLINK-14957))
+The Flink CLI no longer supports the deprecated command line options
+`-yst/--yarnstreaming`, which were used to disable eager pre-allocation of memory.
+All Flink users are advised to remove this command line option.
+
+#### Mesos Integration will reject expired Offers faster ([FLINK-14029](https://issues.apache.org/jira/browse/FLINK-14029))
+Flink's Mesos integration now rejects all expired offers instead of only 4.
+This improves the situation where Fenzo holds on to a lot of expired offers
+without giving them back to the Mesos resource manager.
+
+#### Scheduler Rearchitecture ([FLINK-14651](https://issues.apache.org/jira/browse/FLINK-14651))
+Flink's scheduler was refactored with the goal of making scheduling strategies
+customizable in the future. Using the legacy scheduler is discouraged as it will
+be removed in a future release. However, users that experience issues related to
+scheduling can fallback to the legacy scheduler by setting
+`jobmanager.scheduler` to `legacy` in their `flink-conf.yaml` for the time
+being. Note, however, that using the legacy scheduler with the [Pipelined Region
+Failover Strategy]({{ site.baseurl
+}}/dev/task_failure_recovery.html#restart-pipelined-region-failover-strategy)
+enabled has the following caveats:
+
+* Exceptions that caused a job to restart will not be shown on the job overview page of the Web UI ([FLINK-15917](https://issues.apache.org/jira/browse/FLINK-15917)).
+However, exceptions that cause a job to fail (e.g., when all restart attempts exhausted) will still be shown.
+* The `uptime` metric will not be reset after restarting a job due to task failure ([FLINK-15918](https://issues.apache.org/jira/browse/FLINK-15918)).
+
+Note that in the default `flink-conf.yaml`, the Pipelined Region Failover
+Strategy is already enabled. That is, users that want to use the legacy
+scheduler and cannot accept aforementioned caveats should make sure that
+`jobmanager.execution.failover-strategy` is set to `full` or not set at all.
+
+#### Java 11 Support ([FLINK-10725](https://issues.apache.org/jira/browse/FLINK-10725))
+Beginning from this release, Flink can be compiled and run with Java 11. All
+Java 8 artifacts can be also used with Java 11. This means that users that want
+to run Flink with Java 11 do not have to compile Flink themselves.
+
+When starting Flink with Java 11, the following warnings may be logged:
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by org.apache.flink.core.memory.MemoryUtils (file:/opt/flink/flink-1.10.0/lib/flink-dist_2.11-1.10.0.jar) to constructor java.nio.DirectByteBuffer(long,int)
+ WARNING: Please consider reporting this to the maintainers of org.apache.flink.core.memory.MemoryUtils
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by org.apache.flink.api.java.ClosureCleaner (file:/home/flinkuser/.m2/repository/org/apache/flink/flink-core/1.10.0/flink-core-1.10.0.jar) to field java.lang.String.value
+ WARNING: Please consider reporting this to the maintainers of org.apache.flink.api.java.ClosureCleaner
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by org.jboss.netty.util.internal.ByteBufferUtil (file:/home/flinkuser/.m2/repository/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar) to method java.nio.DirectByteBuffer.cleaner()
+ WARNING: Please consider reporting this to the maintainers of org.jboss.netty.util.internal.ByteBufferUtil
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by com.esotericsoftware.kryo.util.UnsafeUtil (file:/home/flinkuser/.m2/repository/com/esotericsoftware/kryo/kryo/2.24.0/kryo-2.24.0.jar) to constructor java.nio.DirectByteBuffer(long,int,java.lang.Object)
+ WARNING: Please consider reporting this to the maintainers of com.esotericsoftware.kryo.util.UnsafeUtil
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+These warnings are considered harmless and will be addressed in future Flink
+releases.
+
+Lastly, note that the connectors for Cassandra, Hive, HBase, and Kafka 0.8--0.11
+have not been tested with Java 11 because the respective projects did not
+provide Java 11 support at the time of the Flink 1.10.0 release.
+
+
+### Memory Management
+#### New Task Executor Memory Model ([FLINK-13980](https://issues.apache.org/jira/browse/FLINK-13980))
+With
+[FLIP-49](https://cwiki.apache.org/confluence/display/FLINK/FLIP-49%3A+Unified+Memory+Configuration+for+TaskExecutors),
+a new memory model has been introduced for the task executor. New configuration
+options have been introduced to control the memory consumption of the task
+executor process. This affects all types of deployments: standalone, YARN,
+Mesos, and the new active Kubernetes integration. The memory model of the job
+manager process has not been changed yet but it is planned to be updated as
+well.
+
+If you try to reuse your previous Flink configuration without any adjustments,
+the new memory model can result in differently computed memory parameters for
+the JVM and, thus, performance changes.
+
+Please, check [the user documentation](../ops/memory/mem_setup.html) for more details.
+
+##### Deprecation and breaking changes
+The following options have been removed and have no effect anymore:
+
+
+
+
+
Deprecated/removed config option
+
Note
+
+
+
+
+
taskmanager.memory.fraction
+
+ Check also the description of the new option
+ taskmanager.memory.managed.fraction
+ but it has different semantics and the value of the deprecated option
+ usually has to be adjusted
+
+
+
+
taskmanager.memory.off-heap
+
Support for on-heap managed memory has been removed, leaving off-heap managed memory as the only possibility
+
+
+
taskmanager.memory.preallocate
+
Pre-allocation is no longer supported, and managed memory is always allocated lazily
+
+
+
+
+
+The following options, if used, are interpreted as other new options in order to
+maintain backwards compatibility where it makes sense:
+
+
+
+
+
Deprecated config option
+
Interpreted as
+
+
+
+
+
taskmanager.heap.size
+
+
+
taskmanager.memory.flink.size for standalone deployment
+
taskmanager.memory.process.size for containerized deployments
+
+
+
+
+
taskmanager.memory.size
+
taskmanager.memory.managed.size
+
+
+
taskmanager.network.memory.min
+
taskmanager.memory.network.min
+
+
+
taskmanager.network.memory.max
+
taskmanager.memory.network.max
+
+
+
taskmanager.network.memory.fraction
+
taskmanager.memory.network.fraction
+
+
+
+
+
+The container cut-off configuration options, `containerized.heap-cutoff-ratio`
+and `containerized.heap-cutoff-min`, have no effect for task executor processes
+anymore but they still have the same semantics for the JobManager process.
+
+#### RocksDB State Backend Memory Control ([FLINK-7289](https://issues.apache.org/jira/browse/FLINK-7289))
+Together with the introduction of the [new Task Executor Memory
+Model](#new-task-executor-memory-model-flink-13980), the memory consumption of the RocksDB state backend will be
+limited by the total amount of Flink Managed Memory, which can be configured via
+`taskmanager.memory.managed.size` or `taskmanager.memory.managed.fraction`.
+Furthermore, users can tune RocksDB's write/read memory ratio
+(`state.backend.rocksdb.memory.write-buffer-ratio`, by default `0.5`) and the
+reserved memory fraction for indices/filters
+(`state.backend.rocksdb.memory.high-prio-pool-ratio`, by default `0.1`). More
+details and advanced configuration options can be found in the [Flink user
+documentation]({{ site.baseurl }}/ops/state/large_state_tuning.html#tuning-rocksdb-memory).
+
+#### Fine-grained Operator Resource Management ([FLINK-14058](https://issues.apache.org/jira/browse/FLINK-14058))
+Config options `table.exec.resource.external-buffer-memory`,
+`table.exec.resource.hash-agg.memory`, `table.exec.resource.hash-join.memory`,
+and `table.exec.resource.sort.memory` have been deprecated. Beginning from Flink
+1.10, these config options are interpreted as weight hints instead of absolute
+memory requirements. Flink choses sensible default weight hints which should
+not be adjustment by users.
+
+
+### Table API & SQL
+#### Rename of ANY Type to RAW Type ([FLINK-14904](https://issues.apache.org/jira/browse/FLINK-14904))
+The identifier `raw` is a reserved keyword now and must be escaped with
+backticks when used as a SQL field or function name.
+
+#### Rename of Table Connector Properties ([FLINK-14649](https://issues.apache.org/jira/browse/FLINK-14649))
+Some indexed properties for table connectors have been flattened and renamed
+for a better user experience when writing DDL statements. This affects the
+Kafka Connector properties `connector.properties` and
+`connector.specific-offsets`. Furthermore, the Elasticsearch Connector
+property `connector.hosts` is affected. The aforementioned, old properties are
+deprecated and will be removed in future versions. Please consult the [Table
+Connectors documentation]({{ site.baseurl }}/dev/table/connect.html#table-connectors)
+for the new property names.
+
+#### Methods for interacting with temporary Tables & Views ([FLINK-14490](https://issues.apache.org/jira/browse/FLINK-14490))
+Methods `registerTable()`/`registerDataStream()`/`registerDataSet()` have been
+deprecated in favor of `createTemporaryView()`, which better adheres to the
+corresponding SQL term.
+
+The `scan()` method has been deprecated in favor of the `from()` method.
+
+Methods `registerTableSource()`/`registerTableSink()` become deprecated in favor
+of `ConnectTableDescriptor#createTemporaryTable()`. The `ConnectTableDescriptor`
+approach expects only a set of string properties as a description of a
+TableSource or TableSink instead of an instance of a class in case of the
+deprecated methods. This in return makes it possible to reliably store those
+definitions in catalogs.
+
+Method `insertInto(String path, String... pathContinued)` has been removed in
+favor of in `insertInto(String path)`.
+
+All the newly introduced methods accept a String identifier which will be
+parsed into a 3-part identifier. The parser supports quoting the identifier.
+It also requires escaping any reserved SQL keywords.
+
+#### Removal of ExternalCatalog API ([FLINK-13697](https://issues.apache.org/jira/browse/FLINK-13697))
+The deprecated `ExternalCatalog` API has been dropped. This includes:
+
+* `ExternalCatalog` (and all dependent classes, e.g., `ExternalTable`)
+* `SchematicDescriptor`, `MetadataDescriptor`, `StatisticsDescriptor`
+
+Users are advised to use the [new Catalog API]({{ site.baseurl }}/dev/table/catalogs.html#catalog-api).
+
+
+### Configuration
+#### Introduction of Type Information for ConfigOptions ([FLINK-14493](https://issues.apache.org/jira/browse/FLINK-14493))
+Getters of `org.apache.flink.configuration.Configuration` throw
+`IllegalArgumentException` now if the configured value cannot be parsed into
+the required type. In previous Flink releases the default value was returned
+in such cases.
+
+#### Increase of default Restart Delay ([FLINK-13884](https://issues.apache.org/jira/browse/FLINK-13884))
+The default restart delay for all shipped restart strategies, i.e., `fixed-delay`
+and `failure-rate`, has been raised to 1 s (from originally 0 s).
+
+#### Simplification of Cluster-Level Restart Strategy Configuration ([FLINK-13921](https://issues.apache.org/jira/browse/FLINK-13921))
+Previously, if the user had set `restart-strategy.fixed-delay.attempts` or
+`restart-strategy.fixed-delay.delay` but had not configured the option
+`restart-strategy`, the cluster-level restart strategy would have been
+`fixed-delay`. Now the cluster-level restart strategy is only determined by
+the config option `restart-strategy` and whether checkpointing is enabled. See
+[_"Task Failure Recovery"_]({{ site.baseurl }}/dev/task_failure_recovery.html)
+for details.
+
+#### Disable memory-mapped BoundedBlockingSubpartition by default ([FLINK-14952](https://issues.apache.org/jira/browse/FLINK-14952))
+The config option `taskmanager.network.bounded-blocking-subpartition-type` has
+been renamed to `taskmanager.network.blocking-shuffle.type`. Moreover, the
+default value of the aforementioned config option has been changed from `auto`
+to `file`. The reason is that TaskManagers running on YARN with `auto`, could
+easily exceed the memory budget of their container, due to incorrectly accounted
+memory-mapped files memory usage.
+
+#### Removal of non-credit-based Network Flow Control ([FLINK-14516](https://issues.apache.org/jira/browse/FLINK-14516))
+The non-credit-based network flow control code was removed alongside of the
+configuration option `taskmanager.network.credit-model`. Flink will now always
+use credit-based flow control.
+
+#### Removal of HighAvailabilityOptions#HA_JOB_DELAY ([FLINK-13885](https://issues.apache.org/jira/browse/FLINK-13885))
+The configuration option `high-availability.job.delay` has been removed
+since it is no longer used.
+
+
+### State
+#### Enable Background Cleanup of State with TTL by default ([FLINK-14898](https://issues.apache.org/jira/browse/FLINK-14898))
+[Background cleanup of expired state with TTL]({{ site.baseurl }}/dev/stream/state/state.html#cleanup-of-expired-state)
+is activated by default now for all state backends shipped with Flink.
+Note that the RocksDB state backend implements background cleanup by employing
+a compaction filter. This has the caveat that even if a Flink job does not
+store state with TTL, a minor performance penalty during compaction is incurred.
+Users that experience noticeable performance degradation during RocksDB
+compaction can disable the TTL compaction filter by setting the config option
+`state.backend.rocksdb.ttl.compaction.filter.enabled` to `false`.
+
+#### Deprecation of StateTtlConfig#Builder#cleanupInBackground() ([FLINK-15606](https://issues.apache.org/jira/browse/FLINK-15606))
+`StateTtlConfig#Builder#cleanupInBackground()` has been deprecated because the
+background cleanup of state with TTL is already enabled by default.
+
+#### Timers are stored in RocksDB by default when using RocksDBStateBackend ([FLINK-15637](https://issues.apache.org/jira/browse/FLINK-15637))
+The default timer store has been changed from Heap to RocksDB for the RocksDB
+state backend to support asynchronous snapshots for timer state and better
+scalability, with less than 5% performance cost. Users that find the performance
+decline critical can set `state.backend.rocksdb.timer-service.factory` to `HEAP`
+in `flink-conf.yaml` to restore the old behavior.
+
+#### Removal of StateTtlConfig#TimeCharacteristic ([FLINK-15605](https://issues.apache.org/jira/browse/FLINK-15605))
+`StateTtlConfig#TimeCharacteristic` has been removed in favor of
+`StateTtlConfig#TtlTimeCharacteristic`.
+
+#### New efficient Method to check if MapState is empty ([FLINK-13034](https://issues.apache.org/jira/browse/FLINK-13034))
+We have added a new method `MapState#isEmpty()` which enables users to check
+whether a map state is empty. The new method is 40% faster than
+`mapState.keys().iterator().hasNext()` when using the RocksDB state backend.
+
+#### RocksDB Upgrade ([FLINK-14483](https://issues.apache.org/jira/browse/FLINK-14483))
+We have again released our own RocksDB build (FRocksDB) which is based on
+RocksDB version 5.17.2 with several feature backports for the [Write Buffer
+Manager](https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager) to
+enable limiting RocksDB's memory usage. The decision to release our own
+RocksDB build was made because later RocksDB versions suffer from a
+[performance regression under certain
+workloads](https://github.com/facebook/rocksdb/issues/5774).
+
+#### RocksDB Logging disabled by default ([FLINK-15068](https://issues.apache.org/jira/browse/FLINK-15068))
+Logging in RocksDB (e.g., logging related to flush, compaction, memtable
+creation, etc.) has been disabled by default to prevent disk space from being
+filled up unexpectedly. Users that need to enable logging should implement their
+own `RocksDBOptionsFactory` that creates `DBOptions` instances with
+`InfoLogLevel` set to `INFO_LEVEL`.
+
+#### Improved RocksDB Savepoint Recovery ([FLINK-12785](https://issues.apache.org/jira/browse/FLINK-12785))
+In previous Flink releases users may encounter an `OutOfMemoryError` when
+restoring from a RocksDB savepoint containing large KV pairs. For that reason
+we introduced a configurable memory limit in the `RocksDBWriteBatchWrapper`
+with a default value of 2 MB. RocksDB's WriteBatch will flush before the
+consumed memory limit is reached. If needed, the limit can be tuned via the
+`state.backend.rocksdb.write-batch-size` config option in `flink-conf.yaml`.
+
+
+### PyFlink
+#### Python 2 Support dropped ([FLINK-14469](https://issues.apache.org/jira/browse/FLINK-14469))
+Beginning from this release, PyFlink does not support Python 2. This is because [Python 2 has
+reached end of life on January 1,
+2020](https://www.python.org/doc/sunset-python-2/), and several third-party
+projects that PyFlink depends on are also dropping Python 2 support.
+
+
+### Monitoring
+#### InfluxdbReporter skips Inf and NaN ([FLINK-12147](https://issues.apache.org/jira/browse/FLINK-12147))
+The `InfluxdbReporter` now silently skips values that are unsupported by
+InfluxDB, such as `Double.POSITIVE_INFINITY`, `Double.NEGATIVE_INFINITY`,
+`Double.NaN`, etc.
+
+
+### Connectors
+#### Kinesis Connector License Change ([FLINK-12847](https://issues.apache.org/jira/browse/FLINK-12847))
+flink-connector-kinesis is now licensed under the Apache License, Version 2.0,
+and its artifacts will be deployed to Maven central as part of the Flink
+releases. Users no longer need to build the Kinesis connector from source themselves.
+
+
+### Miscellaneous Interface Changes
+#### ExecutionConfig#getGlobalJobParameters() cannot return null anymore ([FLINK-9787](https://issues.apache.org/jira/browse/FLINK-9787))
+`ExecutionConfig#getGlobalJobParameters` has been changed to never return
+`null`. Conversely,
+`ExecutionConfig#setGlobalJobParameters(GlobalJobParameters)` will not accept
+`null` values anymore.
+
+#### Change of contract in MasterTriggerRestoreHook interface ([FLINK-14344](https://issues.apache.org/jira/browse/FLINK-14344))
+Implementations of `MasterTriggerRestoreHook#triggerCheckpoint(long, long,
+Executor)` must be non-blocking now. Any blocking operation should be executed
+asynchronously, e.g., using the given executor.
+
+#### Client-/ and Server-Side Separation of HA Services ([FLINK-13750](https://issues.apache.org/jira/browse/FLINK-13750))
+The `HighAvailabilityServices` have been split up into client-side
+`ClientHighAvailabilityServices` and cluster-side `HighAvailabilityServices`.
+When implementing custom high availability services, users should follow this
+separation by overriding the factory method
+`HighAvailabilityServicesFactory#createClientHAServices(Configuration)`.
+Moreover, `HighAvailabilityServices#getWebMonitorLeaderRetriever()` should no
+longer be implemented since it has been deprecated.
+
+#### Deprecation of HighAvailabilityServices#getWebMonitorLeaderElectionService() ([FLINK-13977](https://issues.apache.org/jira/browse/FLINK-13977))
+Implementations of `HighAvailabilityServices` should implement
+`HighAvailabilityServices#getClusterRestEndpointLeaderElectionService()` instead
+of `HighAvailabilityServices#getWebMonitorLeaderElectionService()`.
+
+#### Interface Change in LeaderElectionService ([FLINK-14287](https://issues.apache.org/jira/browse/FLINK-14287))
+`LeaderElectionService#confirmLeadership(UUID, String)` now takes an
+additional second argument, which is the address under which the leader will be
+reachable. All custom `LeaderElectionService` implementations will need to be
+updated accordingly.
+
+#### Deprecation of Checkpoint Lock ([FLINK-14857](https://issues.apache.org/jira/browse/FLINK-14857))
+The method
+`org.apache.flink.streaming.runtime.tasks.StreamTask#getCheckpointLock()` is
+deprecated now. Users should use `MailboxExecutor` to run actions that require
+synchronization with the task's thread (e.g. collecting output produced by an
+external thread). The methods `MailboxExecutor#yield()` or
+`MailboxExecutor#tryYield()` can be used for actions that need to give up
+control to other actions temporarily, e.g., if the current operator is
+blocked. The `MailboxExecutor` can be accessed by using
+`YieldingOperatorFactory` (see `AsyncWaitOperator` for an example usage).
+
+#### Deprecation of OptionsFactory and ConfigurableOptionsFactory interfaces ([FLINK-14926](https://issues.apache.org/jira/browse/FLINK-14926))
+Interfaces `OptionsFactory` and `ConfigurableOptionsFactory` have been
+deprecated in favor of `RocksDBOptionsFactory` and
+`ConfigurableRocksDBOptionsFactory`, respectively.
diff --git a/docs/release-notes/flink-1.10.zh.md b/docs/release-notes/flink-1.10.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..0e228ec0f500cc51bdf32704042a5900f27707eb
--- /dev/null
+++ b/docs/release-notes/flink-1.10.zh.md
@@ -0,0 +1,466 @@
+---
+title: "Release Notes - Flink 1.10"
+---
+
+
+
+These release notes discuss important aspects, such as configuration, behavior,
+or dependencies, that changed between Flink 1.9 and Flink 1.10. Please read
+these notes carefully if you are planning to upgrade your Flink version to 1.10.
+
+* This will be replaced by the TOC
+{:toc}
+
+
+### Clusters & Deployment
+#### FileSystems should be loaded via Plugin Architecture ([FLINK-11956](https://issues.apache.org/jira/browse/FLINK-11956))
+s3-hadoop and s3-presto filesystems do no longer use class relocations and need
+to be loaded through [plugins]({{ site.baseurl }}/ops/filesystems/#pluggable-file-systems)
+but now seamlessly integrate with all credential providers. Other filesystems
+are strongly recommended to be only used as plugins as we will continue to
+remove relocations.
+
+#### Flink Client respects Classloading Policy ([FLINK-13749](https://issues.apache.org/jira/browse/FLINK-13749))
+The Flink client now also respects the configured classloading policy, i.e.,
+`parent-first` or `child-first` classloading. Previously, only cluster
+components such as the job manager or task manager supported this setting.
+This does mean that users might get different behaviour in their programs, in
+which case they should configure the classloading policy explicitly to use
+`parent-first` classloading, which was the previous (hard-coded) behaviour.
+
+#### Enable spreading out Tasks evenly across all TaskManagers ([FLINK-12122](https://issues.apache.org/jira/browse/FLINK-12122))
+When
+[FLIP-6](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077)
+was rolled out with Flink 1.5.0, we changed how slots are allocated from
+TaskManagers (TMs). Instead of evenly allocating the slots from all registered
+TMs, we had the tendency to exhaust a TM before using another one. To use a
+scheduling strategy that is more similar to the pre-FLIP-6 behaviour, where
+Flink tries to spread out the workload across all currently available TMs, one
+can set `cluster.evenly-spread-out-slots: true` in the `flink-conf.yaml`.
+
+#### Directory Structure Change for highly available Artifacts ([FLINK-13633](https://issues.apache.org/jira/browse/FLINK-13633))
+All highly available artifacts stored by Flink will now be stored under
+`HA_STORAGE_DIR/HA_CLUSTER_ID` with `HA_STORAGE_DIR` configured by
+`high-availability.storageDir` and `HA_CLUSTER_ID` configured by
+`high-availability.cluster-id`.
+
+#### Resources and JARs shipped via --yarnship will be ordered in the Classpath ([FLINK-13127](https://issues.apache.org/jira/browse/FLINK-13127))
+When using the `--yarnship` command line option, resource directories and jar
+files will be added to the classpath in lexicographical order with resources
+directories appearing first.
+
+#### Removal of --yn/--yarncontainer Command Line Options ([FLINK-12362](https://issues.apache.org/jira/browse/FLINK-12362))
+The Flink CLI no longer supports the deprecated command line options
+`-yn/--yarncontainer`, which were used to specify the number of containers to
+start on YARN. This option has been deprecated since the introduction of
+[FLIP-6](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077).
+All Flink users are advised to remove this command line option.
+
+#### Removal of --yst/--yarnstreaming Command Line Options ([FLINK-14957](https://issues.apache.org/jira/browse/FLINK-14957))
+The Flink CLI no longer supports the deprecated command line options
+`-yst/--yarnstreaming`, which were used to disable eager pre-allocation of memory.
+All Flink users are advised to remove this command line option.
+
+#### Mesos Integration will reject expired Offers faster ([FLINK-14029](https://issues.apache.org/jira/browse/FLINK-14029))
+Flink's Mesos integration now rejects all expired offers instead of only 4.
+This improves the situation where Fenzo holds on to a lot of expired offers
+without giving them back to the Mesos resource manager.
+
+#### Scheduler Rearchitecture ([FLINK-14651](https://issues.apache.org/jira/browse/FLINK-14651))
+Flink's scheduler was refactored with the goal of making scheduling strategies
+customizable in the future. Using the legacy scheduler is discouraged as it will
+be removed in a future release. However, users that experience issues related to
+scheduling can fallback to the legacy scheduler by setting
+`jobmanager.scheduler` to `legacy` in their `flink-conf.yaml` for the time
+being. Note, however, that using the legacy scheduler with the [Pipelined Region
+Failover Strategy]({{ site.baseurl
+}}/dev/task_failure_recovery.html#restart-pipelined-region-failover-strategy)
+enabled has the following caveats:
+
+* Exceptions that caused a job to restart will not be shown on the job overview page of the Web UI ([FLINK-15917](https://issues.apache.org/jira/browse/FLINK-15917)).
+However, exceptions that cause a job to fail (e.g., when all restart attempts exhausted) will still be shown.
+* The `uptime` metric will not be reset after restarting a job due to task failure ([FLINK-15918](https://issues.apache.org/jira/browse/FLINK-15918)).
+
+Note that in the default `flink-conf.yaml`, the Pipelined Region Failover
+Strategy is already enabled. That is, users that want to use the legacy
+scheduler and cannot accept aforementioned caveats should make sure that
+`jobmanager.execution.failover-strategy` is set to `full` or not set at all.
+
+#### Java 11 Support ([FLINK-10725](https://issues.apache.org/jira/browse/FLINK-10725))
+Beginning from this release, Flink can be compiled and run with Java 11. All
+Java 8 artifacts can be also used with Java 11. This means that users that want
+to run Flink with Java 11 do not have to compile Flink themselves.
+
+When starting Flink with Java 11, the following warnings may be logged:
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by org.apache.flink.core.memory.MemoryUtils (file:/opt/flink/flink-1.10.0/lib/flink-dist_2.11-1.10.0.jar) to constructor java.nio.DirectByteBuffer(long,int)
+ WARNING: Please consider reporting this to the maintainers of org.apache.flink.core.memory.MemoryUtils
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by org.apache.flink.api.java.ClosureCleaner (file:/home/flinkuser/.m2/repository/org/apache/flink/flink-core/1.10.0/flink-core-1.10.0.jar) to field java.lang.String.value
+ WARNING: Please consider reporting this to the maintainers of org.apache.flink.api.java.ClosureCleaner
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by org.jboss.netty.util.internal.ByteBufferUtil (file:/home/flinkuser/.m2/repository/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar) to method java.nio.DirectByteBuffer.cleaner()
+ WARNING: Please consider reporting this to the maintainers of org.jboss.netty.util.internal.ByteBufferUtil
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+ WARNING: An illegal reflective access operation has occurred
+ WARNING: Illegal reflective access by com.esotericsoftware.kryo.util.UnsafeUtil (file:/home/flinkuser/.m2/repository/com/esotericsoftware/kryo/kryo/2.24.0/kryo-2.24.0.jar) to constructor java.nio.DirectByteBuffer(long,int,java.lang.Object)
+ WARNING: Please consider reporting this to the maintainers of com.esotericsoftware.kryo.util.UnsafeUtil
+ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+ WARNING: All illegal access operations will be denied in a future release
+
+These warnings are considered harmless and will be addressed in future Flink
+releases.
+
+Lastly, note that the connectors for Cassandra, Hive, HBase, and Kafka 0.8--0.11
+have not been tested with Java 11 because the respective projects did not
+provide Java 11 support at the time of the Flink 1.10.0 release.
+
+
+### Memory Management
+#### New Task Executor Memory Model ([FLINK-13980](https://issues.apache.org/jira/browse/FLINK-13980))
+With
+[FLIP-49](https://cwiki.apache.org/confluence/display/FLINK/FLIP-49%3A+Unified+Memory+Configuration+for+TaskExecutors),
+a new memory model has been introduced for the task executor. New configuration
+options have been introduced to control the memory consumption of the task
+executor process. This affects all types of deployments: standalone, YARN,
+Mesos, and the new active Kubernetes integration. The memory model of the job
+manager process has not been changed yet but it is planned to be updated as
+well.
+
+If you try to reuse your previous Flink configuration without any adjustments,
+the new memory model can result in differently computed memory parameters for
+the JVM and, thus, performance changes.
+
+Please, check [the user documentation](../ops/memory/mem_setup.html) for more details.
+
+##### Deprecation and breaking changes
+The following options have been removed and have no effect anymore:
+
+
+
+
+
Deprecated/removed config option
+
Note
+
+
+
+
+
taskmanager.memory.fraction
+
+ Check also the description of the new option
+ taskmanager.memory.managed.fraction
+ but it has different semantics and the value of the deprecated option
+ usually has to be adjusted
+
+
+
+
taskmanager.memory.off-heap
+
Support for on-heap managed memory has been removed, leaving off-heap managed memory as the only possibility
+
+
+
taskmanager.memory.preallocate
+
Pre-allocation is no longer supported, and managed memory is always allocated lazily
+
+
+
+
+
+The following options, if used, are interpreted as other new options in order to
+maintain backwards compatibility where it makes sense:
+
+
+
+
+
Deprecated config option
+
Interpreted as
+
+
+
+
+
taskmanager.heap.size
+
+
+
taskmanager.memory.flink.size for standalone deployment
+
taskmanager.memory.process.size for containerized deployments
+
+
+
+
+
taskmanager.memory.size
+
taskmanager.memory.managed.size
+
+
+
taskmanager.network.memory.min
+
taskmanager.memory.network.min
+
+
+
taskmanager.network.memory.max
+
taskmanager.memory.network.max
+
+
+
taskmanager.network.memory.fraction
+
taskmanager.memory.network.fraction
+
+
+
+
+
+The container cut-off configuration options, `containerized.heap-cutoff-ratio`
+and `containerized.heap-cutoff-min`, have no effect for task executor processes
+anymore but they still have the same semantics for the JobManager process.
+
+#### RocksDB State Backend Memory Control ([FLINK-7289](https://issues.apache.org/jira/browse/FLINK-7289))
+Together with the introduction of the [new Task Executor Memory
+Model](#new-task-executor-memory-model-flink-13980), the memory consumption of the RocksDB state backend will be
+limited by the total amount of Flink Managed Memory, which can be configured via
+`taskmanager.memory.managed.size` or `taskmanager.memory.managed.fraction`.
+Furthermore, users can tune RocksDB's write/read memory ratio
+(`state.backend.rocksdb.memory.write-buffer-ratio`, by default `0.5`) and the
+reserved memory fraction for indices/filters
+(`state.backend.rocksdb.memory.high-prio-pool-ratio`, by default `0.1`). More
+details and advanced configuration options can be found in the [Flink user
+documentation]({{ site.baseurl }}/ops/state/large_state_tuning.html#tuning-rocksdb-memory).
+
+#### Fine-grained Operator Resource Management ([FLINK-14058](https://issues.apache.org/jira/browse/FLINK-14058))
+Config options `table.exec.resource.external-buffer-memory`,
+`table.exec.resource.hash-agg.memory`, `table.exec.resource.hash-join.memory`,
+and `table.exec.resource.sort.memory` have been deprecated. Beginning from Flink
+1.10, these config options are interpreted as weight hints instead of absolute
+memory requirements. Flink choses sensible default weight hints which should
+not be adjustment by users.
+
+
+### Table API & SQL
+#### Rename of ANY Type to RAW Type ([FLINK-14904](https://issues.apache.org/jira/browse/FLINK-14904))
+The identifier `raw` is a reserved keyword now and must be escaped with
+backticks when used as a SQL field or function name.
+
+#### Rename of Table Connector Properties ([FLINK-14649](https://issues.apache.org/jira/browse/FLINK-14649))
+Some indexed properties for table connectors have been flattened and renamed
+for a better user experience when writing DDL statements. This affects the
+Kafka Connector properties `connector.properties` and
+`connector.specific-offsets`. Furthermore, the Elasticsearch Connector
+property `connector.hosts` is affected. The aforementioned, old properties are
+deprecated and will be removed in future versions. Please consult the [Table
+Connectors documentation]({{ site.baseurl }}/dev/table/connect.html#table-connectors)
+for the new property names.
+
+#### Methods for interacting with temporary Tables & Views ([FLINK-14490](https://issues.apache.org/jira/browse/FLINK-14490))
+Methods `registerTable()`/`registerDataStream()`/`registerDataSet()` have been
+deprecated in favor of `createTemporaryView()`, which better adheres to the
+corresponding SQL term.
+
+The `scan()` method has been deprecated in favor of the `from()` method.
+
+Methods `registerTableSource()`/`registerTableSink()` become deprecated in favor
+of `ConnectTableDescriptor#createTemporaryTable()`. The `ConnectTableDescriptor`
+approach expects only a set of string properties as a description of a
+TableSource or TableSink instead of an instance of a class in case of the
+deprecated methods. This in return makes it possible to reliably store those
+definitions in catalogs.
+
+Method `insertInto(String path, String... pathContinued)` has been removed in
+favor of in `insertInto(String path)`.
+
+All the newly introduced methods accept a String identifier which will be
+parsed into a 3-part identifier. The parser supports quoting the identifier.
+It also requires escaping any reserved SQL keywords.
+
+#### Removal of ExternalCatalog API ([FLINK-13697](https://issues.apache.org/jira/browse/FLINK-13697))
+The deprecated `ExternalCatalog` API has been dropped. This includes:
+
+* `ExternalCatalog` (and all dependent classes, e.g., `ExternalTable`)
+* `SchematicDescriptor`, `MetadataDescriptor`, `StatisticsDescriptor`
+
+Users are advised to use the [new Catalog API]({{ site.baseurl }}/dev/table/catalogs.html#catalog-api).
+
+
+### Configuration
+#### Introduction of Type Information for ConfigOptions ([FLINK-14493](https://issues.apache.org/jira/browse/FLINK-14493))
+Getters of `org.apache.flink.configuration.Configuration` throw
+`IllegalArgumentException` now if the configured value cannot be parsed into
+the required type. In previous Flink releases the default value was returned
+in such cases.
+
+#### Increase of default Restart Delay ([FLINK-13884](https://issues.apache.org/jira/browse/FLINK-13884))
+The default restart delay for all shipped restart strategies, i.e., `fixed-delay`
+and `failure-rate`, has been raised to 1 s (from originally 0 s).
+
+#### Simplification of Cluster-Level Restart Strategy Configuration ([FLINK-13921](https://issues.apache.org/jira/browse/FLINK-13921))
+Previously, if the user had set `restart-strategy.fixed-delay.attempts` or
+`restart-strategy.fixed-delay.delay` but had not configured the option
+`restart-strategy`, the cluster-level restart strategy would have been
+`fixed-delay`. Now the cluster-level restart strategy is only determined by
+the config option `restart-strategy` and whether checkpointing is enabled. See
+[_"Task Failure Recovery"_]({{ site.baseurl }}/dev/task_failure_recovery.html)
+for details.
+
+#### Disable memory-mapped BoundedBlockingSubpartition by default ([FLINK-14952](https://issues.apache.org/jira/browse/FLINK-14952))
+The config option `taskmanager.network.bounded-blocking-subpartition-type` has
+been renamed to `taskmanager.network.blocking-shuffle.type`. Moreover, the
+default value of the aforementioned config option has been changed from `auto`
+to `file`. The reason is that TaskManagers running on YARN with `auto`, could
+easily exceed the memory budget of their container, due to incorrectly accounted
+memory-mapped files memory usage.
+
+#### Removal of non-credit-based Network Flow Control ([FLINK-14516](https://issues.apache.org/jira/browse/FLINK-14516))
+The non-credit-based network flow control code was removed alongside of the
+configuration option `taskmanager.network.credit-model`. Flink will now always
+use credit-based flow control.
+
+#### Removal of HighAvailabilityOptions#HA_JOB_DELAY ([FLINK-13885](https://issues.apache.org/jira/browse/FLINK-13885))
+The configuration option `high-availability.job.delay` has been removed
+since it is no longer used.
+
+
+### State
+#### Enable Background Cleanup of State with TTL by default ([FLINK-14898](https://issues.apache.org/jira/browse/FLINK-14898))
+[Background cleanup of expired state with TTL]({{ site.baseurl }}/dev/stream/state/state.html#cleanup-of-expired-state)
+is activated by default now for all state backends shipped with Flink.
+Note that the RocksDB state backend implements background cleanup by employing
+a compaction filter. This has the caveat that even if a Flink job does not
+store state with TTL, a minor performance penalty during compaction is incurred.
+Users that experience noticeable performance degradation during RocksDB
+compaction can disable the TTL compaction filter by setting the config option
+`state.backend.rocksdb.ttl.compaction.filter.enabled` to `false`.
+
+#### Deprecation of StateTtlConfig#Builder#cleanupInBackground() ([FLINK-15606](https://issues.apache.org/jira/browse/FLINK-15606))
+`StateTtlConfig#Builder#cleanupInBackground()` has been deprecated because the
+background cleanup of state with TTL is already enabled by default.
+
+#### Timers are stored in RocksDB by default when using RocksDBStateBackend ([FLINK-15637](https://issues.apache.org/jira/browse/FLINK-15637))
+The default timer store has been changed from Heap to RocksDB for the RocksDB
+state backend to support asynchronous snapshots for timer state and better
+scalability, with less than 5% performance cost. Users that find the performance
+decline critical can set `state.backend.rocksdb.timer-service.factory` to `HEAP`
+in `flink-conf.yaml` to restore the old behavior.
+
+#### Removal of StateTtlConfig#TimeCharacteristic ([FLINK-15605](https://issues.apache.org/jira/browse/FLINK-15605))
+`StateTtlConfig#TimeCharacteristic` has been removed in favor of
+`StateTtlConfig#TtlTimeCharacteristic`.
+
+#### New efficient Method to check if MapState is empty ([FLINK-13034](https://issues.apache.org/jira/browse/FLINK-13034))
+We have added a new method `MapState#isEmpty()` which enables users to check
+whether a map state is empty. The new method is 40% faster than
+`mapState.keys().iterator().hasNext()` when using the RocksDB state backend.
+
+#### RocksDB Upgrade ([FLINK-14483](https://issues.apache.org/jira/browse/FLINK-14483))
+We have again released our own RocksDB build (FRocksDB) which is based on
+RocksDB version 5.17.2 with several feature backports for the [Write Buffer
+Manager](https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager) to
+enable limiting RocksDB's memory usage. The decision to release our own
+RocksDB build was made because later RocksDB versions suffer from a
+[performance regression under certain
+workloads](https://github.com/facebook/rocksdb/issues/5774).
+
+#### RocksDB Logging disabled by default ([FLINK-15068](https://issues.apache.org/jira/browse/FLINK-15068))
+Logging in RocksDB (e.g., logging related to flush, compaction, memtable
+creation, etc.) has been disabled by default to prevent disk space from being
+filled up unexpectedly. Users that need to enable logging should implement their
+own `RocksDBOptionsFactory` that creates `DBOptions` instances with
+`InfoLogLevel` set to `INFO_LEVEL`.
+
+#### Improved RocksDB Savepoint Recovery ([FLINK-12785](https://issues.apache.org/jira/browse/FLINK-12785))
+In previous Flink releases users may encounter an `OutOfMemoryError` when
+restoring from a RocksDB savepoint containing large KV pairs. For that reason
+we introduced a configurable memory limit in the `RocksDBWriteBatchWrapper`
+with a default value of 2 MB. RocksDB's WriteBatch will flush before the
+consumed memory limit is reached. If needed, the limit can be tuned via the
+`state.backend.rocksdb.write-batch-size` config option in `flink-conf.yaml`.
+
+
+### PyFlink
+#### Python 2 Support dropped ([FLINK-14469](https://issues.apache.org/jira/browse/FLINK-14469))
+Beginning from this release, PyFlink does not support Python 2. This is because [Python 2 has
+reached end of life on January 1,
+2020](https://www.python.org/doc/sunset-python-2/), and several third-party
+projects that PyFlink depends on are also dropping Python 2 support.
+
+
+### Monitoring
+#### InfluxdbReporter skips Inf and NaN ([FLINK-12147](https://issues.apache.org/jira/browse/FLINK-12147))
+The `InfluxdbReporter` now silently skips values that are unsupported by
+InfluxDB, such as `Double.POSITIVE_INFINITY`, `Double.NEGATIVE_INFINITY`,
+`Double.NaN`, etc.
+
+
+### Connectors
+#### Kinesis Connector License Change ([FLINK-12847](https://issues.apache.org/jira/browse/FLINK-12847))
+flink-connector-kinesis is now licensed under the Apache License, Version 2.0,
+and its artifacts will be deployed to Maven central as part of the Flink
+releases. Users no longer need to build the Kinesis connector from source themselves.
+
+
+### Miscellaneous Interface Changes
+#### ExecutionConfig#getGlobalJobParameters() cannot return null anymore ([FLINK-9787](https://issues.apache.org/jira/browse/FLINK-9787))
+`ExecutionConfig#getGlobalJobParameters` has been changed to never return
+`null`. Conversely,
+`ExecutionConfig#setGlobalJobParameters(GlobalJobParameters)` will not accept
+`null` values anymore.
+
+#### Change of contract in MasterTriggerRestoreHook interface ([FLINK-14344](https://issues.apache.org/jira/browse/FLINK-14344))
+Implementations of `MasterTriggerRestoreHook#triggerCheckpoint(long, long,
+Executor)` must be non-blocking now. Any blocking operation should be executed
+asynchronously, e.g., using the given executor.
+
+#### Client-/ and Server-Side Separation of HA Services ([FLINK-13750](https://issues.apache.org/jira/browse/FLINK-13750))
+The `HighAvailabilityServices` have been split up into client-side
+`ClientHighAvailabilityServices` and cluster-side `HighAvailabilityServices`.
+When implementing custom high availability services, users should follow this
+separation by overriding the factory method
+`HighAvailabilityServicesFactory#createClientHAServices(Configuration)`.
+Moreover, `HighAvailabilityServices#getWebMonitorLeaderRetriever()` should no
+longer be implemented since it has been deprecated.
+
+#### Deprecation of HighAvailabilityServices#getWebMonitorLeaderElectionService() ([FLINK-13977](https://issues.apache.org/jira/browse/FLINK-13977))
+Implementations of `HighAvailabilityServices` should implement
+`HighAvailabilityServices#getClusterRestEndpointLeaderElectionService()` instead
+of `HighAvailabilityServices#getWebMonitorLeaderElectionService()`.
+
+#### Interface Change in LeaderElectionService ([FLINK-14287](https://issues.apache.org/jira/browse/FLINK-14287))
+`LeaderElectionService#confirmLeadership(UUID, String)` now takes an
+additional second argument, which is the address under which the leader will be
+reachable. All custom `LeaderElectionService` implementations will need to be
+updated accordingly.
+
+#### Deprecation of Checkpoint Lock ([FLINK-14857](https://issues.apache.org/jira/browse/FLINK-14857))
+The method
+`org.apache.flink.streaming.runtime.tasks.StreamTask#getCheckpointLock()` is
+deprecated now. Users should use `MailboxExecutor` to run actions that require
+synchronization with the task's thread (e.g. collecting output produced by an
+external thread). The methods `MailboxExecutor#yield()` or
+`MailboxExecutor#tryYield()` can be used for actions that need to give up
+control to other actions temporarily, e.g., if the current operator is
+blocked. The `MailboxExecutor` can be accessed by using
+`YieldingOperatorFactory` (see `AsyncWaitOperator` for an example usage).
+
+#### Deprecation of OptionsFactory and ConfigurableOptionsFactory interfaces ([FLINK-14926](https://issues.apache.org/jira/browse/FLINK-14926))
+Interfaces `OptionsFactory` and `ConfigurableOptionsFactory` have been
+deprecated in favor of `RocksDBOptionsFactory` and
+`ConfigurableRocksDBOptionsFactory`, respectively.
diff --git a/docs/release-notes/flink-1.8.md b/docs/release-notes/flink-1.8.md
index c9fbdad89585e215d4c2cae1b7cc0d8719a7f864..713b11173b2db46ca666baa543f63c25500c3d97 100644
--- a/docs/release-notes/flink-1.8.md
+++ b/docs/release-notes/flink-1.8.md
@@ -200,7 +200,14 @@ The `CompositeSerializerSnapshot` utility class has been removed. You should
now use `CompositeTypeSerializerSnapshot` instead, for snapshots of composite
serializers that delegate serialization to multiple nested serializers. Please
see
-[here](/dev/stream/state/custom_serialization.html#implementing-a-compositetypeserializersnapshot)
+[here](http://ci.apache.org/projects/flink/flink-docs-release-1.8/dev/stream/state/custom_serialization.html#implementing-a-compositetypeserializersnapshot)
for instructions on using `CompositeTypeSerializerSnapshot`.
+### Memory management
+
+In Fink 1.8.0 and prior version, the managed memory fraction of taskmanager is controlled by `taskmanager.memory.fraction`,
+and with 0.7 as the default value. However, sometimes this will cause OOMs due to the fact that the default value of JVM
+parameter `NewRatio` is 2, which means the old generation occupied only 2/3 (0.66) of the heap memory. So if you run into
+this case, please manually change this value to a lower value.
+
{% top %}
diff --git a/docs/release-notes/flink-1.8.zh.md b/docs/release-notes/flink-1.8.zh.md
index c9fbdad89585e215d4c2cae1b7cc0d8719a7f864..713b11173b2db46ca666baa543f63c25500c3d97 100644
--- a/docs/release-notes/flink-1.8.zh.md
+++ b/docs/release-notes/flink-1.8.zh.md
@@ -200,7 +200,14 @@ The `CompositeSerializerSnapshot` utility class has been removed. You should
now use `CompositeTypeSerializerSnapshot` instead, for snapshots of composite
serializers that delegate serialization to multiple nested serializers. Please
see
-[here](/dev/stream/state/custom_serialization.html#implementing-a-compositetypeserializersnapshot)
+[here](http://ci.apache.org/projects/flink/flink-docs-release-1.8/dev/stream/state/custom_serialization.html#implementing-a-compositetypeserializersnapshot)
for instructions on using `CompositeTypeSerializerSnapshot`.
+### Memory management
+
+In Fink 1.8.0 and prior version, the managed memory fraction of taskmanager is controlled by `taskmanager.memory.fraction`,
+and with 0.7 as the default value. However, sometimes this will cause OOMs due to the fact that the default value of JVM
+parameter `NewRatio` is 2, which means the old generation occupied only 2/3 (0.66) of the heap memory. So if you run into
+this case, please manually change this value to a lower value.
+
{% top %}
diff --git a/docs/release-notes/flink-1.9.md b/docs/release-notes/flink-1.9.md
index dedfa4c832696a85947beb8690c82cdf6142a4f7..d0523a191d99c699031f3c8bff4e3ec42708f249 100644
--- a/docs/release-notes/flink-1.9.md
+++ b/docs/release-notes/flink-1.9.md
@@ -75,6 +75,16 @@ have skipped Java 9 support.
Related issues:
- [FLINK-8033: JDK 9 support](https://issues.apache.org/jira/browse/FLINK-8033)
+### Memory management
+
+In Fink 1.9.0 and prior version, the managed memory fraction of taskmanager is controlled by `taskmanager.memory.fraction`,
+and with 0.7 as the default value. However, sometimes this will cause OOMs due to the fact that the default value of JVM
+parameter `NewRatio` is 2, which means the old generation occupied only 2/3 (0.66) of the heap memory. So if you run into
+this case, please manually change this value to a lower value.
+
+Related issues:
+- [FLINK-14123: Lower the default value of taskmanager.memory.fraction](https://issues.apache.org/jira/browse/FLINK-14123)
+
## Deprecations and breaking changes
### Scala expression DSL for Table API moved to `flink-table-api-scala`
diff --git a/docs/release-notes/flink-1.9.zh.md b/docs/release-notes/flink-1.9.zh.md
index dedfa4c832696a85947beb8690c82cdf6142a4f7..d0523a191d99c699031f3c8bff4e3ec42708f249 100644
--- a/docs/release-notes/flink-1.9.zh.md
+++ b/docs/release-notes/flink-1.9.zh.md
@@ -75,6 +75,16 @@ have skipped Java 9 support.
Related issues:
- [FLINK-8033: JDK 9 support](https://issues.apache.org/jira/browse/FLINK-8033)
+### Memory management
+
+In Fink 1.9.0 and prior version, the managed memory fraction of taskmanager is controlled by `taskmanager.memory.fraction`,
+and with 0.7 as the default value. However, sometimes this will cause OOMs due to the fact that the default value of JVM
+parameter `NewRatio` is 2, which means the old generation occupied only 2/3 (0.66) of the heap memory. So if you run into
+this case, please manually change this value to a lower value.
+
+Related issues:
+- [FLINK-14123: Lower the default value of taskmanager.memory.fraction](https://issues.apache.org/jira/browse/FLINK-14123)
+
## Deprecations and breaking changes
### Scala expression DSL for Table API moved to `flink-table-api-scala`
diff --git a/flink-annotations/pom.xml b/flink-annotations/pom.xml
index d43e8027ad83e65e4fa2ec122d251d89944c1ed9..f6189a062b7c0561ca75849d41e1f36dedac86d2 100644
--- a/flink-annotations/pom.xml
+++ b/flink-annotations/pom.xml
@@ -25,7 +25,7 @@ under the License.
org.apache.flinkflink-parent
- 1.10-SNAPSHOT
+ 1.11-SNAPSHOT..
diff --git a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
index 988404931128747b1c7386999ed457710f905783..e7dbce3f8ac5cc529421778ed0ed40678a7860db 100644
--- a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
+++ b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
@@ -41,24 +41,65 @@ public final class Documentation {
}
/**
- * Annotation used on config option fields to include them in the "Common Options" section.
+ * Annotation used on config option fields to include them in specific sections. Sections are groups of options
+ * that are aggregated across option classes, with each group being placed into a dedicated file.
*
- *
The {@link CommonOption#position()} argument controls the position in the generated table, with lower values
+ *
The {@link Section#position()} argument controls the position in the generated table, with lower values
* being placed at the top. Fields with the same position are sorted alphabetically by key.
*/
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
@Internal
- public @interface CommonOption {
- int POSITION_MEMORY = 10;
- int POSITION_PARALLELISM_SLOTS = 20;
- int POSITION_FAULT_TOLERANCE = 30;
- int POSITION_HIGH_AVAILABILITY = 40;
- int POSITION_SECURITY = 50;
+ public @interface Section {
+ /**
+ * The sections in the config docs where this option should be included.
+ */
+ String[] value() default {};
+
+ /**
+ * The relative position of the option in its section.
+ */
int position() default Integer.MAX_VALUE;
}
+ /**
+ * Constants for section names.
+ */
+ public static final class Sections {
+
+ public static final String COMMON_HOST_PORT = "common_host_port";
+ public static final String COMMON_STATE_BACKENDS = "common_state_backends";
+ public static final String COMMON_HIGH_AVAILABILITY = "common_high_availability";
+ public static final String COMMON_HIGH_AVAILABILITY_ZOOKEEPER = "common_high_availability_zk";
+ public static final String COMMON_MEMORY = "common_memory";
+ public static final String COMMON_MISCELLANEOUS = "common_miscellaneous";
+
+ public static final String SECURITY_SSL = "security_ssl";
+ public static final String SECURITY_AUTH_KERBEROS = "security_auth_kerberos";
+ public static final String SECURITY_AUTH_ZOOKEEPER = "security_auth_zk";
+
+ public static final String STATE_BACKEND_ROCKSDB = "state_backend_rocksdb";
+
+ public static final String EXPERT_CLASS_LOADING = "expert_class_loading";
+ public static final String EXPERT_SCHEDULING = "expert_scheduling";
+ public static final String EXPERT_FAULT_TOLERANCE = "expert_fault_tolerance";
+ public static final String EXPERT_STATE_BACKENDS = "expert_state_backends";
+ public static final String EXPERT_REST = "expert_rest";
+ public static final String EXPERT_HIGH_AVAILABILITY = "expert_high_availability";
+ public static final String EXPERT_ZOOKEEPER_HIGH_AVAILABILITY = "expert_high_availability_zk";
+ public static final String EXPERT_SECURITY_SSL = "expert_security_ssl";
+ public static final String EXPERT_ROCKSDB = "expert_rocksdb";
+
+ public static final String ALL_JOB_MANAGER = "all_jobmanager";
+ public static final String ALL_TASK_MANAGER = "all_taskmanager";
+ public static final String ALL_TASK_MANAGER_NETWORK = "all_taskmanager_network";
+
+ public static final String DEPRECATED_FILE_SINKS = "deprecated_file_sinks";
+
+ private Sections() {}
+ }
+
/**
* Annotation used on table config options for adding meta data labels.
*
@@ -104,6 +145,16 @@ public final class Documentation {
String value() default "";
}
+ /**
+ * Annotation used on config option fields or options class to mark them as a suffix-option; i.e., a config option
+ * where the key is only a suffix, with the prefix being danymically provided at runtime.
+ */
+ @Target({ElementType.FIELD, ElementType.TYPE})
+ @Retention(RetentionPolicy.RUNTIME)
+ @Internal
+ public @interface SuffixOption {
+ }
+
private Documentation(){
}
}
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index ba6dc5fe0408cc399d396523b87bfe989b7a8dcb..1481b6eb997ffcd8423f03ca8a5127da29c813e5 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -26,7 +26,7 @@ under the License.
org.apache.flinkflink-parent
- 1.10-SNAPSHOT
+ 1.11-SNAPSHOT..
diff --git a/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java b/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java
index 9fb4ce5b9f7244b6f0b65042f8a9299e2a1b63c2..3c6515db9eaa2f428f9291d5cefb440438c1988d 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java
@@ -18,16 +18,32 @@
package org.apache.flink.client;
-import org.apache.flink.core.fs.Path;
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.ContextEnvironment;
+import org.apache.flink.client.program.ContextEnvironmentFactory;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.client.program.ProgramInvocationException;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.core.execution.DetachedJobExecutionResult;
+import org.apache.flink.core.execution.PipelineExecutorServiceLoader;
+import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmaster.JobResult;
+import org.apache.flink.util.ExceptionUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.File;
import java.io.IOException;
-import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
-import java.util.jar.JarFile;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Utility functions for Flink client.
@@ -35,49 +51,96 @@ import java.util.jar.JarFile;
public enum ClientUtils {
;
- /**
- * Adds the given jar files to the {@link JobGraph} via {@link JobGraph#addJar}. This will
- * throw an exception if a jar URL is not valid.
- */
- public static void addJarFiles(JobGraph jobGraph, List jarFilesToAttach) {
- for (URL jar : jarFilesToAttach) {
- try {
- jobGraph.addJar(new Path(jar.toURI()));
- } catch (URISyntaxException e) {
- throw new RuntimeException("URL is invalid. This should not happen.", e);
- }
+ private static final Logger LOG = LoggerFactory.getLogger(ClientUtils.class);
+
+ public static ClassLoader buildUserCodeClassLoader(
+ List jars,
+ List classpaths,
+ ClassLoader parent,
+ Configuration configuration) {
+ URL[] urls = new URL[jars.size() + classpaths.size()];
+ for (int i = 0; i < jars.size(); i++) {
+ urls[i] = jars.get(i);
}
+ for (int i = 0; i < classpaths.size(); i++) {
+ urls[i + jars.size()] = classpaths.get(i);
+ }
+ final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);
+ final String classLoaderResolveOrder =
+ configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);
+ FlinkUserCodeClassLoaders.ResolveOrder resolveOrder =
+ FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder);
+ return FlinkUserCodeClassLoaders.create(resolveOrder, urls, parent, alwaysParentFirstLoaderPatterns);
}
- public static void checkJarFile(URL jar) throws IOException {
- File jarFile;
+ public static JobExecutionResult submitJob(
+ ClusterClient client,
+ JobGraph jobGraph) throws ProgramInvocationException {
+ checkNotNull(client);
+ checkNotNull(jobGraph);
try {
- jarFile = new File(jar.toURI());
- } catch (URISyntaxException e) {
- throw new IOException("JAR file path is invalid '" + jar + '\'');
- }
- if (!jarFile.exists()) {
- throw new IOException("JAR file does not exist '" + jarFile.getAbsolutePath() + '\'');
+ return client
+ .submitJob(jobGraph)
+ .thenApply(DetachedJobExecutionResult::new)
+ .get();
+ } catch (InterruptedException | ExecutionException e) {
+ ExceptionUtils.checkInterrupted(e);
+ throw new ProgramInvocationException("Could not run job in detached mode.", jobGraph.getJobID(), e);
}
- if (!jarFile.canRead()) {
- throw new IOException("JAR file can't be read '" + jarFile.getAbsolutePath() + '\'');
+ }
+
+ public static JobExecutionResult submitJobAndWaitForResult(
+ ClusterClient client,
+ JobGraph jobGraph,
+ ClassLoader classLoader) throws ProgramInvocationException {
+ checkNotNull(client);
+ checkNotNull(jobGraph);
+ checkNotNull(classLoader);
+
+ JobResult jobResult;
+
+ try {
+ jobResult = client
+ .submitJob(jobGraph)
+ .thenCompose(client::requestJobResult)
+ .get();
+ } catch (InterruptedException | ExecutionException e) {
+ ExceptionUtils.checkInterrupted(e);
+ throw new ProgramInvocationException("Could not run job", jobGraph.getJobID(), e);
}
- try (JarFile ignored = new JarFile(jarFile)) {
- // verify that we can open the Jar file
- } catch (IOException e) {
- throw new IOException("Error while opening jar file '" + jarFile.getAbsolutePath() + '\'', e);
+ try {
+ return jobResult.toJobExecutionResult(classLoader);
+ } catch (JobExecutionException | IOException | ClassNotFoundException e) {
+ throw new ProgramInvocationException("Job failed", jobGraph.getJobID(), e);
}
}
- public static ClassLoader buildUserCodeClassLoader(List jars, List classpaths, ClassLoader parent) {
- URL[] urls = new URL[jars.size() + classpaths.size()];
- for (int i = 0; i < jars.size(); i++) {
- urls[i] = jars.get(i);
- }
- for (int i = 0; i < classpaths.size(); i++) {
- urls[i + jars.size()] = classpaths.get(i);
+ public static void executeProgram(
+ PipelineExecutorServiceLoader executorServiceLoader,
+ Configuration configuration,
+ PackagedProgram program) throws ProgramInvocationException {
+ checkNotNull(executorServiceLoader);
+ final ClassLoader userCodeClassLoader = program.getUserCodeClassLoader();
+ final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(userCodeClassLoader);
+
+ LOG.info("Starting program (detached: {})", !configuration.getBoolean(DeploymentOptions.ATTACHED));
+
+ ContextEnvironmentFactory factory = new ContextEnvironmentFactory(
+ executorServiceLoader,
+ configuration,
+ userCodeClassLoader);
+ ContextEnvironment.setAsContext(factory);
+
+ try {
+ program.invokeInteractiveModeForExecution();
+ } finally {
+ ContextEnvironment.unsetContext();
+ }
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
}
- return FlinkUserCodeClassLoaders.parentFirst(urls, parent);
}
}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/RemoteExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/RemoteExecutor.java
deleted file mode 100644
index 91a5d3a930f4066b0acb0fd0bb65fd3c876bd21b..0000000000000000000000000000000000000000
--- a/flink-clients/src/main/java/org/apache/flink/client/RemoteExecutor.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.client;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.api.common.PlanExecutor;
-import org.apache.flink.api.dag.Pipeline;
-import org.apache.flink.client.program.ClusterClient;
-import org.apache.flink.client.program.rest.RestClusterClient;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.JobManagerOptions;
-import org.apache.flink.configuration.RestOptions;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-
-import java.net.InetSocketAddress;
-import java.net.URL;
-import java.util.List;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * The RemoteExecutor is a {@link org.apache.flink.api.common.PlanExecutor} that takes the program
- * and ships it to a remote Flink cluster for execution.
- *
- *
The RemoteExecutor is pointed at the JobManager and gets the program and (if necessary) the
- * set of libraries that need to be shipped together with the program.
- *
- *
The RemoteExecutor is used in the {@link org.apache.flink.api.java.RemoteEnvironment} to
- * remotely execute program parts.
- */
-public class RemoteExecutor extends PlanExecutor {
-
- private final Configuration clientConfiguration;
-
- private int defaultParallelism = 1;
-
- public RemoteExecutor(String hostname, int port) {
- this(hostname, port, new Configuration());
- }
-
- public RemoteExecutor(String hostname, int port, Configuration clientConfiguration) {
- this(new InetSocketAddress(hostname, port), clientConfiguration);
- }
-
- public RemoteExecutor(InetSocketAddress inet, Configuration clientConfiguration) {
- this.clientConfiguration = clientConfiguration;
-
- clientConfiguration.setString(JobManagerOptions.ADDRESS, inet.getHostName());
- clientConfiguration.setInteger(JobManagerOptions.PORT, inet.getPort());
- clientConfiguration.setInteger(RestOptions.PORT, inet.getPort());
- }
-
- // ------------------------------------------------------------------------
- // Properties
- // ------------------------------------------------------------------------
-
- /**
- * Sets the parallelism that will be used when neither the program does not define any
- * parallelism at all.
- *
- * @param defaultParallelism The default parallelism for the executor.
- */
- public void setDefaultParallelism(int defaultParallelism) {
- if (defaultParallelism < 1) {
- throw new IllegalArgumentException("The default parallelism must be at least one");
- }
- this.defaultParallelism = defaultParallelism;
- }
-
- /**
- * Gets the parallelism that will be used when neither the program does not define any
- * parallelism at all.
- *
- * @return The default parallelism for the executor.
- */
- public int getDefaultParallelism() {
- return defaultParallelism;
- }
-
- // ------------------------------------------------------------------------
- // Executing programs
- // ------------------------------------------------------------------------
-
- @Override
- public JobExecutionResult executePlan(
- Pipeline plan,
- List jarFiles,
- List globalClasspaths) throws Exception {
- checkNotNull(plan);
-
- JobGraph jobGraph = FlinkPipelineTranslationUtil.getJobGraph(plan,
- clientConfiguration,
- getDefaultParallelism());
-
- ClientUtils.addJarFiles(jobGraph, jarFiles);
- jobGraph.setClasspaths(globalClasspaths);
-
- ClassLoader userCodeClassLoader = ClientUtils.buildUserCodeClassLoader(
- jarFiles,
- globalClasspaths,
- getClass().getClassLoader());
-
- return executePlanWithJars(jobGraph, userCodeClassLoader);
- }
-
- private JobExecutionResult executePlanWithJars(JobGraph jobGraph, ClassLoader classLoader) throws Exception {
- checkNotNull(jobGraph);
- checkNotNull(classLoader);
-
- try (ClusterClient client = new RestClusterClient<>(
- clientConfiguration,
- "RemoteExecutor")) {
- return client.submitJob(jobGraph, classLoader).getJobExecutionResult();
- }
- }
-}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java b/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java
index 0cae6e72e6f97b6b8b25c6881dc3a8921da4fc0e..ab539e5a7a1bd44d3e4c8e574fc800c27feded41 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java
@@ -18,7 +18,9 @@
package org.apache.flink.client.cli;
+import org.apache.flink.client.deployment.executors.RemoteExecutor;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
import org.apache.flink.configuration.HighAvailabilityOptions;
import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.util.FlinkException;
@@ -26,8 +28,10 @@ import org.apache.flink.util.NetUtils;
import org.apache.flink.util.Preconditions;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
import java.net.InetSocketAddress;
@@ -38,7 +42,7 @@ import static org.apache.flink.client.cli.CliFrontend.setJobManagerAddressInConf
* a ZooKeeper namespace.
*
*/
-public abstract class AbstractCustomCommandLine implements CustomCommandLine {
+public abstract class AbstractCustomCommandLine implements CustomCommandLine {
protected final Option zookeeperNamespaceOption = new Option("z", "zookeeperNamespace", true,
"Namespace to create the Zookeeper sub-paths for high availability mode");
@@ -69,14 +73,10 @@ public abstract class AbstractCustomCommandLine implements CustomCommandLine<
baseOptions.addOption(zookeeperNamespaceOption);
}
- /**
- * Override configuration settings by specified command line options.
- *
- * @param commandLine containing the overriding values
- * @return Effective configuration with the overridden configuration settings
- */
- protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
+ @Override
+ public Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
final Configuration resultingConfiguration = new Configuration(configuration);
+ resultingConfiguration.setString(DeploymentOptions.TARGET, RemoteExecutor.NAME);
if (commandLine.hasOption(addressOption.getOpt())) {
String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
@@ -91,4 +91,38 @@ public abstract class AbstractCustomCommandLine implements CustomCommandLine<
return resultingConfiguration;
}
+
+ protected void printUsage() {
+ System.out.println("Usage:");
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.setWidth(200);
+ formatter.setLeftPadding(5);
+
+ formatter.setSyntaxPrefix(" Optional");
+ Options options = new Options();
+ addGeneralOptions(options);
+ addRunOptions(options);
+ formatter.printHelp(" ", options);
+ }
+
+ public static int handleCliArgsException(CliArgsException e, Logger logger) {
+ logger.error("Could not parse the command line arguments.", e);
+
+ System.out.println(e.getMessage());
+ System.out.println();
+ System.out.println("Use the help option (-h or --help) to get help on the command.");
+ return 1;
+ }
+
+ public static int handleError(Throwable t, Logger logger) {
+ logger.error("Error while running the Flink session.", t);
+
+ System.err.println();
+ System.err.println("------------------------------------------------------------");
+ System.err.println(" The program finished with the following exception:");
+ System.err.println();
+
+ t.printStackTrace();
+ return 1;
+ }
}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
index 552eccc7f96a1463d8a53ab31558e0af7c57e39a..f688c42bb0e74dc41d16bb5fe4fea6d70b9cded2 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
@@ -20,14 +20,15 @@ package org.apache.flink.client.cli;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobSubmissionResult;
-import org.apache.flink.api.common.accumulators.AccumulatorHelper;
+import org.apache.flink.api.common.JobStatus;
import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.ClientUtils;
import org.apache.flink.client.FlinkPipelineTranslationUtil;
+import org.apache.flink.client.deployment.ClusterClientFactory;
+import org.apache.flink.client.deployment.ClusterClientServiceLoader;
import org.apache.flink.client.deployment.ClusterDescriptor;
-import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.client.program.PackagedProgram;
import org.apache.flink.client.program.PackagedProgramUtils;
@@ -40,20 +41,17 @@ import org.apache.flink.configuration.CoreOptions;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.core.execution.DefaultExecutorServiceLoader;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.plugin.PluginUtils;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.client.JobStatusMessage;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.security.SecurityConfiguration;
import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.flink.runtime.util.EnvironmentInformation;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.ShutdownHookUtil;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
@@ -63,7 +61,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.InetSocketAddress;
import java.net.URL;
@@ -80,6 +77,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import static org.apache.flink.client.cli.CliFrontendParser.HELP_OPTION;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Implementation of a simple command line frontend for executing programs.
*/
@@ -103,7 +103,7 @@ public class CliFrontend {
private final Configuration configuration;
- private final List> customCommandLines;
+ private final List customCommandLines;
private final Options customCommandLineOptions;
@@ -111,17 +111,27 @@ public class CliFrontend {
private final int defaultParallelism;
+ private final ClusterClientServiceLoader clusterClientServiceLoader;
+
public CliFrontend(
Configuration configuration,
- List> customCommandLines) {
- this.configuration = Preconditions.checkNotNull(configuration);
- this.customCommandLines = Preconditions.checkNotNull(customCommandLines);
+ List customCommandLines) {
+ this(configuration, new DefaultClusterClientServiceLoader(), customCommandLines);
+ }
+
+ public CliFrontend(
+ Configuration configuration,
+ ClusterClientServiceLoader clusterClientServiceLoader,
+ List customCommandLines) {
+ this.configuration = checkNotNull(configuration);
+ this.customCommandLines = checkNotNull(customCommandLines);
+ this.clusterClientServiceLoader = checkNotNull(clusterClientServiceLoader);
FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));
this.customCommandLineOptions = new Options();
- for (CustomCommandLine customCommandLine : customCommandLines) {
+ for (CustomCommandLine customCommandLine : customCommandLines) {
customCommandLine.addGeneralOptions(customCommandLineOptions);
customCommandLine.addRunOptions(customCommandLineOptions);
}
@@ -169,17 +179,17 @@ public class CliFrontend {
final CommandLine commandLine = CliFrontendParser.parse(commandLineOptions, args, true);
- final RunOptions runOptions = new RunOptions(commandLine);
+ final ProgramOptions programOptions = new ProgramOptions(commandLine);
// evaluate help flag
- if (runOptions.isPrintHelp()) {
+ if (commandLine.hasOption(HELP_OPTION.getOpt())) {
CliFrontendParser.printHelpForRun(customCommandLines);
return;
}
- if (!runOptions.isPython()) {
+ if (!programOptions.isPython()) {
// Java program should be specified a JAR file
- if (runOptions.getJarFilePath() == null) {
+ if (programOptions.getJarFilePath() == null) {
throw new CliArgsException("Java program should be specified a JAR file.");
}
}
@@ -187,110 +197,38 @@ public class CliFrontend {
final PackagedProgram program;
try {
LOG.info("Building program from JAR file");
- program = buildProgram(runOptions);
+ program = buildProgram(programOptions);
}
catch (FileNotFoundException e) {
- throw new CliArgsException("Could not build the program from JAR file.", e);
+ throw new CliArgsException("Could not build the program from JAR file: " + e.getMessage(), e);
}
- final CustomCommandLine customCommandLine = getActiveCustomCommandLine(commandLine);
+ final List jobJars = program.getJobJarAndDependencies();
+ final Configuration effectiveConfiguration =
+ getEffectiveConfiguration(commandLine, programOptions, jobJars);
+
+ LOG.debug("Effective executor configuration: {}", effectiveConfiguration);
try {
- runProgram(customCommandLine, commandLine, runOptions, program);
+ executeProgram(effectiveConfiguration, program);
} finally {
program.deleteExtractedLibraries();
}
}
- private void runProgram(
- CustomCommandLine customCommandLine,
- CommandLine commandLine,
- RunOptions runOptions,
- PackagedProgram program) throws ProgramInvocationException, FlinkException {
- final ClusterDescriptor clusterDescriptor = customCommandLine.createClusterDescriptor(commandLine);
-
- try {
- final T clusterId = customCommandLine.getClusterId(commandLine);
+ private Configuration getEffectiveConfiguration(
+ final CommandLine commandLine,
+ final ProgramOptions programOptions,
+ final List jobJars) throws FlinkException {
- final ClusterClient client;
+ final CustomCommandLine customCommandLine = getActiveCustomCommandLine(checkNotNull(commandLine));
+ final ExecutionConfigAccessor executionParameters = ExecutionConfigAccessor.fromProgramOptions(
+ checkNotNull(programOptions),
+ checkNotNull(jobJars));
- // directly deploy the job if the cluster is started in job mode and detached
- if (clusterId == null && runOptions.getDetachedMode()) {
- int parallelism = runOptions.getParallelism() == -1 ? defaultParallelism : runOptions.getParallelism();
-
- final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, configuration, parallelism);
-
- final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
- client = clusterDescriptor.deployJobCluster(
- clusterSpecification,
- jobGraph,
- runOptions.getDetachedMode());
-
- logAndSysout("Job has been submitted with JobID " + jobGraph.getJobID());
-
- try {
- client.close();
- } catch (Exception e) {
- LOG.info("Could not properly shut down the client.", e);
- }
- } else {
- final Thread shutdownHook;
- if (clusterId != null) {
- client = clusterDescriptor.retrieve(clusterId);
- shutdownHook = null;
- } else {
- // also in job mode we have to deploy a session cluster because the job
- // might consist of multiple parts (e.g. when using collect)
- final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
- client = clusterDescriptor.deploySessionCluster(clusterSpecification);
- // if not running in detached mode, add a shutdown hook to shut down cluster if client exits
- // there's a race-condition here if cli is killed before shutdown hook is installed
- if (!runOptions.getDetachedMode() && runOptions.isShutdownOnAttachedExit()) {
- shutdownHook = ShutdownHookUtil.addShutdownHook(client::shutDownCluster, client.getClass().getSimpleName(), LOG);
- } else {
- shutdownHook = null;
- }
- }
-
- try {
- client.setDetached(runOptions.getDetachedMode());
-
- LOG.debug("{}", runOptions.getSavepointRestoreSettings());
-
- int userParallelism = runOptions.getParallelism();
- LOG.debug("User parallelism is set to {}", userParallelism);
- if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) {
- userParallelism = defaultParallelism;
- }
-
- executeProgram(program, client, userParallelism);
- } finally {
- if (clusterId == null && !client.isDetached()) {
- // terminate the cluster only if we have started it before and if it's not detached
- try {
- client.shutDownCluster();
- } catch (final Exception e) {
- LOG.info("Could not properly terminate the Flink cluster.", e);
- }
- if (shutdownHook != null) {
- // we do not need the hook anymore as we have just tried to shutdown the cluster.
- ShutdownHookUtil.removeShutdownHook(shutdownHook, client.getClass().getSimpleName(), LOG);
- }
- }
- try {
- client.close();
- } catch (Exception e) {
- LOG.info("Could not properly shut down the client.", e);
- }
- }
- }
- } finally {
- try {
- clusterDescriptor.close();
- } catch (Exception e) {
- LOG.info("Could not properly close the cluster descriptor.", e);
- }
- }
+ final Configuration executorConfig = customCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
+ final Configuration effectiveConfiguration = new Configuration(executorConfig);
+ return executionParameters.applyToConfiguration(effectiveConfiguration);
}
/**
@@ -305,32 +243,32 @@ public class CliFrontend {
final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);
- InfoOptions infoOptions = new InfoOptions(commandLine);
+ final ProgramOptions programOptions = new ProgramOptions(commandLine);
// evaluate help flag
- if (infoOptions.isPrintHelp()) {
+ if (commandLine.hasOption(HELP_OPTION.getOpt())) {
CliFrontendParser.printHelpForInfo();
return;
}
- if (infoOptions.getJarFilePath() == null) {
+ if (programOptions.getJarFilePath() == null) {
throw new CliArgsException("The program JAR file was not specified.");
}
// -------- build the packaged program -------------
LOG.info("Building program from JAR file");
- final PackagedProgram program = buildProgram(infoOptions);
+ final PackagedProgram program = buildProgram(programOptions);
try {
- int parallelism = infoOptions.getParallelism();
+ int parallelism = programOptions.getParallelism();
if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
parallelism = defaultParallelism;
}
LOG.info("Creating program plan dump");
- Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, parallelism);
+ Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, parallelism, true);
String jsonPlan = FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);
if (jsonPlan != null) {
@@ -394,7 +332,7 @@ public class CliFrontend {
showAll = listOptions.showAll();
}
- final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
+ final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
runClusterAction(
activeCommandLine,
@@ -403,8 +341,8 @@ public class CliFrontend {
}
- private void listJobs(
- ClusterClient clusterClient,
+ private void listJobs(
+ ClusterClient clusterClient,
boolean showRunning,
boolean showScheduled,
boolean showAll) throws FlinkException {
@@ -512,14 +450,14 @@ public class CliFrontend {
logAndSysout((advanceToEndOfEventTime ? "Draining job " : "Suspending job ") + "\"" + jobId + "\" with a savepoint.");
- final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
+ final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
runClusterAction(
activeCommandLine,
commandLine,
clusterClient -> {
final String savepointPath;
try {
- savepointPath = clusterClient.stopWithSavepoint(jobId, advanceToEndOfEventTime, targetDirectory);
+ savepointPath = clusterClient.stopWithSavepoint(jobId, advanceToEndOfEventTime, targetDirectory).get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS);
} catch (Exception e) {
throw new FlinkException("Could not stop with a savepoint job \"" + jobId + "\".", e);
}
@@ -549,7 +487,7 @@ public class CliFrontend {
return;
}
- final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
+ final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
final String[] cleanedArgs = cancelOptions.getArgs();
@@ -580,7 +518,7 @@ public class CliFrontend {
clusterClient -> {
final String savepointPath;
try {
- savepointPath = clusterClient.cancelWithSavepoint(jobId, targetDirectory);
+ savepointPath = clusterClient.cancelWithSavepoint(jobId, targetDirectory).get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS);
} catch (Exception e) {
throw new FlinkException("Could not cancel job " + jobId + '.', e);
}
@@ -602,7 +540,7 @@ public class CliFrontend {
commandLine,
clusterClient -> {
try {
- clusterClient.cancel(jobId);
+ clusterClient.cancel(jobId).get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS);
} catch (Exception e) {
throw new FlinkException("Could not cancel job " + jobId + '.', e);
}
@@ -634,7 +572,7 @@ public class CliFrontend {
return;
}
- final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
+ final CustomCommandLine activeCommandLine = getActiveCustomCommandLine(commandLine);
if (savepointOptions.isDispose()) {
runClusterAction(
@@ -678,33 +616,29 @@ public class CliFrontend {
/**
* Sends a SavepointTriggerMessage to the job manager.
*/
- private String triggerSavepoint(ClusterClient clusterClient, JobID jobId, String savepointDirectory) throws FlinkException {
+ private void triggerSavepoint(ClusterClient clusterClient, JobID jobId, String savepointDirectory) throws FlinkException {
logAndSysout("Triggering savepoint for job " + jobId + '.');
+
CompletableFuture savepointPathFuture = clusterClient.triggerSavepoint(jobId, savepointDirectory);
logAndSysout("Waiting for response...");
- final String savepointPath;
-
try {
- savepointPath = savepointPathFuture.get();
- }
- catch (Exception e) {
+ final String savepointPath = savepointPathFuture.get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS);
+
+ logAndSysout("Savepoint completed. Path: " + savepointPath);
+ logAndSysout("You can resume your program from this savepoint with the run command.");
+ } catch (Exception e) {
Throwable cause = ExceptionUtils.stripExecutionException(e);
throw new FlinkException("Triggering a savepoint for the job " + jobId + " failed.", cause);
}
-
- logAndSysout("Savepoint completed. Path: " + savepointPath);
- logAndSysout("You can resume your program from this savepoint with the run command.");
-
- return savepointPath;
}
/**
* Sends a SavepointDisposalRequest to the job manager.
*/
private void disposeSavepoint(ClusterClient clusterClient, String savepointPath) throws FlinkException {
- Preconditions.checkNotNull(savepointPath, "Missing required argument: savepoint path. " +
+ checkNotNull(savepointPath, "Missing required argument: savepoint path. " +
"Usage: bin/flink savepoint -d ");
logAndSysout("Disposing savepoint '" + savepointPath + "'.");
@@ -726,29 +660,8 @@ public class CliFrontend {
// Interaction with programs and JobManager
// --------------------------------------------------------------------------------------------
- protected void executeProgram(PackagedProgram program, ClusterClient client, int parallelism) throws ProgramMissingJobException, ProgramInvocationException {
- logAndSysout("Starting execution of program");
-
- final JobSubmissionResult result = client.run(program, parallelism);
-
- if (null == result) {
- throw new ProgramMissingJobException("No JobSubmissionResult returned, please make sure you called " +
- "ExecutionEnvironment.execute()");
- }
-
- if (result.isJobExecutionResult()) {
- logAndSysout("Program execution finished");
- JobExecutionResult execResult = result.getJobExecutionResult();
- System.out.println("Job with JobID " + execResult.getJobID() + " has finished.");
- System.out.println("Job Runtime: " + execResult.getNetRuntime() + " ms");
- Map accumulatorsResult = execResult.getAllAccumulatorResults();
- if (accumulatorsResult.size() > 0) {
- System.out.println("Accumulator Results: ");
- System.out.println(AccumulatorHelper.getResultsFormatted(accumulatorsResult));
- }
- } else {
- logAndSysout("Job has been submitted with JobID " + result.getJobID());
- }
+ protected void executeProgram(final Configuration configuration, final PackagedProgram program) throws ProgramInvocationException {
+ ClientUtils.executeProgram(DefaultExecutorServiceLoader.INSTANCE, configuration, program);
}
/**
@@ -756,15 +669,15 @@ public class CliFrontend {
*
* @return A PackagedProgram (upon success)
*/
- PackagedProgram buildProgram(ProgramOptions options) throws FileNotFoundException, ProgramInvocationException {
- String[] programArgs = options.getProgramArgs();
- String jarFilePath = options.getJarFilePath();
- List classpaths = options.getClasspaths();
+ PackagedProgram buildProgram(final ProgramOptions runOptions) throws FileNotFoundException, ProgramInvocationException {
+ String[] programArgs = runOptions.getProgramArgs();
+ String jarFilePath = runOptions.getJarFilePath();
+ List classpaths = runOptions.getClasspaths();
// Get assembler class
- String entryPointClass = options.getEntryPointClassName();
+ String entryPointClass = runOptions.getEntryPointClassName();
File jarFile = null;
- if (options.isPython()) {
+ if (runOptions.isPython()) {
// If the job is specified a jar file
if (jarFilePath != null) {
jarFile = getJarFile(jarFilePath);
@@ -782,13 +695,14 @@ public class CliFrontend {
jarFile = getJarFile(jarFilePath);
}
- PackagedProgram program = entryPointClass == null ?
- new PackagedProgram(jarFile, classpaths, programArgs) :
- new PackagedProgram(jarFile, classpaths, entryPointClass, programArgs);
-
- program.setSavepointRestoreSettings(options.getSavepointRestoreSettings());
-
- return program;
+ return PackagedProgram.newBuilder()
+ .setJarFile(jarFile)
+ .setUserClassPaths(classpaths)
+ .setEntryPointClassName(entryPointClass)
+ .setConfiguration(configuration)
+ .setSavepointRestoreSettings(runOptions.getSavepointRestoreSettings())
+ .setArguments(programArgs)
+ .build();
}
/**
@@ -912,36 +826,21 @@ public class CliFrontend {
* @param activeCommandLine to create the {@link ClusterDescriptor} from
* @param commandLine containing the parsed command line options
* @param clusterAction the cluster action to run against the retrieved {@link ClusterClient}.
- * @param type of the cluster id
+ * @param type of the cluster id
* @throws FlinkException if something goes wrong
*/
- private void runClusterAction(CustomCommandLine activeCommandLine, CommandLine commandLine, ClusterAction clusterAction) throws FlinkException {
- final ClusterDescriptor clusterDescriptor = activeCommandLine.createClusterDescriptor(commandLine);
-
- final T clusterId = activeCommandLine.getClusterId(commandLine);
+ private void runClusterAction(CustomCommandLine activeCommandLine, CommandLine commandLine, ClusterAction clusterAction) throws FlinkException {
+ final Configuration executorConfig = activeCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
+ final ClusterClientFactory clusterClientFactory = clusterClientServiceLoader.getClusterClientFactory(executorConfig);
+ final ClusterID clusterId = clusterClientFactory.getClusterId(executorConfig);
if (clusterId == null) {
- throw new FlinkException("No cluster id was specified. Please specify a cluster to which " +
- "you would like to connect.");
- } else {
- try {
- final ClusterClient clusterClient = clusterDescriptor.retrieve(clusterId);
+ throw new FlinkException("No cluster id was specified. Please specify a cluster to which you would like to connect.");
+ }
- try {
- clusterAction.runAction(clusterClient);
- } finally {
- try {
- clusterClient.close();
- } catch (Exception e) {
- LOG.info("Could not properly shut down the cluster client.", e);
- }
- }
- } finally {
- try {
- clusterDescriptor.close();
- } catch (Exception e) {
- LOG.info("Could not properly close the cluster descriptor.", e);
- }
+ try (final ClusterDescriptor clusterDescriptor = clusterClientFactory.createClusterDescriptor(executorConfig)) {
+ try (final ClusterClient clusterClient = clusterDescriptor.retrieve(clusterId).getClusterClient()) {
+ clusterAction.runAction(clusterClient);
}
}
}
@@ -950,10 +849,10 @@ public class CliFrontend {
* Internal interface to encapsulate cluster actions which are executed via
* the {@link ClusterClient}.
*
- * @param type of the cluster id
+ * @param type of the cluster id
*/
@FunctionalInterface
- private interface ClusterAction {
+ private interface ClusterAction {
/**
* Run the cluster action with the given {@link ClusterClient}.
@@ -961,7 +860,7 @@ public class CliFrontend {
* @param clusterClient to run the cluster action against
* @throws FlinkException if something goes wrong
*/
- void runAction(ClusterClient clusterClient) throws FlinkException;
+ void runAction(ClusterClient clusterClient) throws FlinkException;
}
// --------------------------------------------------------------------------------------------
@@ -1055,7 +954,7 @@ public class CliFrontend {
final Configuration configuration = GlobalConfiguration.loadConfiguration(configurationDirectory);
// 3. load the custom command lines
- final List> customCommandLines = loadCustomCommandLines(
+ final List customCommandLines = loadCustomCommandLines(
configuration,
configurationDirectory);
@@ -1120,13 +1019,11 @@ public class CliFrontend {
config.setInteger(RestOptions.PORT, address.getPort());
}
- public static List> loadCustomCommandLines(Configuration configuration, String configurationDirectory) {
- List> customCommandLines = new ArrayList<>(2);
+ public static List loadCustomCommandLines(Configuration configuration, String configurationDirectory) {
+ List customCommandLines = new ArrayList<>();
// Command line interface of the YARN session, with a special initialization here
// to prefix all options with y/yarn.
- // Tips: DefaultCLI must be added at last, because getActiveCustomCommandLine(..) will get the
- // active CustomCommandLine in order and DefaultCLI isActive always return true.
final String flinkYarnSessionCLI = "org.apache.flink.yarn.cli.FlinkYarnSessionCli";
try {
customCommandLines.add(
@@ -1139,6 +1036,10 @@ public class CliFrontend {
LOG.warn("Could not load CLI class {}.", flinkYarnSessionCLI, e);
}
+ customCommandLines.add(new ExecutorCLI(configuration));
+
+ // Tips: DefaultCLI must be added at last, because getActiveCustomCommandLine(..) will get the
+ // active CustomCommandLine in order and DefaultCLI isActive always return true.
customCommandLines.add(new DefaultCLI(configuration));
return customCommandLines;
@@ -1153,8 +1054,10 @@ public class CliFrontend {
* @param commandLine The input to the command-line.
* @return custom command-line which is active (may only be one at a time)
*/
- public CustomCommandLine getActiveCustomCommandLine(CommandLine commandLine) {
- for (CustomCommandLine cli : customCommandLines) {
+ public CustomCommandLine getActiveCustomCommandLine(CommandLine commandLine) {
+ LOG.debug("Custom commandlines: {}", customCommandLines);
+ for (CustomCommandLine cli : customCommandLines) {
+ LOG.debug("Checking custom commandline {}, isActive: {}", cli, cli.isActive(commandLine));
if (cli.isActive(commandLine)) {
return cli;
}
@@ -1167,7 +1070,7 @@ public class CliFrontend {
* @param className The fully-qualified class name to load.
* @param params The constructor parameters
*/
- private static CustomCommandLine loadCustomCommandLine(String className, Object... params) throws IllegalAccessException, InvocationTargetException, InstantiationException, ClassNotFoundException, NoSuchMethodException {
+ private static CustomCommandLine loadCustomCommandLine(String className, Object... params) throws Exception {
Class customCliClass =
Class.forName(className).asSubclass(CustomCommandLine.class);
@@ -1175,7 +1078,7 @@ public class CliFrontend {
// construct class types from the parameters
Class[] types = new Class[params.length];
for (int i = 0; i < params.length; i++) {
- Preconditions.checkNotNull(params[i], "Parameters for custom command-lines may not be null.");
+ checkNotNull(params[i], "Parameters for custom command-lines may not be null.");
types[i] = params[i].getClass();
}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
index 5639cb545cec0eeeca425722307a54055e20107c..1ecaa2a9d159cac6c72588aa3508f9a1c5abea94 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
@@ -44,7 +44,7 @@ public class CliFrontendParser {
static final Option JAR_OPTION = new Option("j", "jarfile", true, "Flink program JAR file.");
static final Option CLASS_OPTION = new Option("c", "class", true,
- "Class with the program entry point (\"main()\" method or \"getPlan()\" method). Only needed if the " +
+ "Class with the program entry point (\"main()\" method). Only needed if the " +
"JAR file does not specify the class in its manifest.");
static final Option CLASSPATH_OPTION = new Option("C", "classpath", true, "Adds a URL to each user code " +
@@ -57,6 +57,10 @@ public class CliFrontendParser {
"The parallelism with which to run the program. Optional flag to override the default value " +
"specified in the configuration.");
+ /**
+ * @deprecated This has no effect anymore, we're keeping it to not break existing bash scripts.
+ */
+ @Deprecated
static final Option LOGGING_OPTION = new Option("q", "sysoutLogging", false, "If present, " +
"suppress logging output to standard out.");
@@ -126,14 +130,42 @@ public class CliFrontendParser {
static final Option PYFILES_OPTION = new Option("pyfs", "pyFiles", true,
"Attach custom python files for job. " +
- "Comma can be used as the separator to specify multiple files. " +
- "The standard python resource file suffixes such as .py/.egg/.zip are all supported." +
- "(eg: --pyFiles file:///tmp/myresource.zip,hdfs:///$namenode_address/myresource2.zip)");
+ "These files will be added to the PYTHONPATH of both the local client and the remote python UDF worker. " +
+ "The standard python resource file suffixes such as .py/.egg/.zip or directory are all supported. " +
+ "Comma (',') could be used as the separator to specify multiple files " +
+ "(e.g.: --pyFiles file:///tmp/myresource.zip,hdfs:///$namenode_address/myresource2.zip).");
static final Option PYMODULE_OPTION = new Option("pym", "pyModule", true,
"Python module with the program entry point. " +
"This option must be used in conjunction with `--pyFiles`.");
+ static final Option PYREQUIREMENTS_OPTION = new Option("pyreq", "pyRequirements", true,
+ "Specify a requirements.txt file which defines the third-party dependencies. " +
+ "These dependencies will be installed and added to the PYTHONPATH of the python UDF worker. " +
+ "A directory which contains the installation packages of these dependencies could be specified " +
+ "optionally. Use '#' as the separator if the optional parameter exists " +
+ "(e.g.: --pyRequirements file:///tmp/requirements.txt#file:///tmp/cached_dir).");
+
+ static final Option PYARCHIVE_OPTION = new Option("pyarch", "pyArchives", true,
+ "Add python archive files for job. The archive files will be extracted to the working directory " +
+ "of python UDF worker. Currently only zip-format is supported. For each archive file, a target directory " +
+ "be specified. If the target directory name is specified, the archive file will be extracted to a " +
+ "name can directory with the specified name. Otherwise, the archive file will be extracted to a " +
+ "directory with the same name of the archive file. The files uploaded via this option are accessible " +
+ "via relative path. '#' could be used as the separator of the archive file path and the target directory " +
+ "name. Comma (',') could be used as the separator to specify multiple archive files. " +
+ "This option can be used to upload the virtual environment, the data files used in Python UDF " +
+ "(e.g.: --pyArchives file:///tmp/py37.zip,file:///tmp/data.zip#data --pyExecutable " +
+ "py37.zip/py37/bin/python). The data files could be accessed in Python UDF, e.g.: " +
+ "f = open('data/data.txt', 'r').");
+
+ static final Option PYEXEC_OPTION = new Option("pyexec", "pyExecutable", true,
+ "Specify the path of the python interpreter used to execute the python UDF worker " +
+ "(e.g.: --pyExecutable /usr/local/bin/python3). " +
+ "The python UDF worker depends on Python 3.5+, Apache Beam (version == 2.19.0), " +
+ "Pip (version >= 7.1.0) and SetupTools (version >= 37.0.0). " +
+ "Please ensure that the specified environment meets the above requirements.");
+
static {
HELP_OPTION.setRequired(false);
@@ -183,16 +215,22 @@ public class CliFrontendParser {
STOP_AND_DRAIN.setRequired(false);
PY_OPTION.setRequired(false);
- PY_OPTION.setArgName("python");
+ PY_OPTION.setArgName("pythonFile");
PYFILES_OPTION.setRequired(false);
- PYFILES_OPTION.setArgName("pyFiles");
+ PYFILES_OPTION.setArgName("pythonFiles");
PYMODULE_OPTION.setRequired(false);
- PYMODULE_OPTION.setArgName("pyModule");
+ PYMODULE_OPTION.setArgName("pythonModule");
+
+ PYREQUIREMENTS_OPTION.setRequired(false);
+
+ PYARCHIVE_OPTION.setRequired(false);
+
+ PYEXEC_OPTION.setRequired(false);
}
- private static final Options RUN_OPTIONS = getRunCommandOptions();
+ static final Options RUN_OPTIONS = getRunCommandOptions();
private static Options buildGeneralOptions(Options options) {
options.addOption(HELP_OPTION);
@@ -214,6 +252,9 @@ public class CliFrontendParser {
options.addOption(PY_OPTION);
options.addOption(PYFILES_OPTION);
options.addOption(PYMODULE_OPTION);
+ options.addOption(PYREQUIREMENTS_OPTION);
+ options.addOption(PYARCHIVE_OPTION);
+ options.addOption(PYEXEC_OPTION);
return options;
}
@@ -221,12 +262,14 @@ public class CliFrontendParser {
options.addOption(CLASS_OPTION);
options.addOption(CLASSPATH_OPTION);
options.addOption(PARALLELISM_OPTION);
- options.addOption(LOGGING_OPTION);
options.addOption(DETACHED_OPTION);
options.addOption(SHUTDOWN_IF_ATTACHED_OPTION);
options.addOption(PY_OPTION);
options.addOption(PYFILES_OPTION);
options.addOption(PYMODULE_OPTION);
+ options.addOption(PYREQUIREMENTS_OPTION);
+ options.addOption(PYARCHIVE_OPTION);
+ options.addOption(PYEXEC_OPTION);
return options;
}
@@ -284,7 +327,9 @@ public class CliFrontendParser {
private static Options getListOptionsWithoutDeprecatedOptions(Options options) {
options.addOption(RUNNING_OPTION);
- return options.addOption(SCHEDULED_OPTION);
+ options.addOption(ALL_OPTION);
+ options.addOption(SCHEDULED_OPTION);
+ return options;
}
private static Options getCancelOptionsWithoutDeprecatedOptions(Options options) {
@@ -306,7 +351,7 @@ public class CliFrontendParser {
/**
* Prints the help for the client.
*/
- public static void printHelp(Collection> customCommandLines) {
+ public static void printHelp(Collection customCommandLines) {
System.out.println("./flink [OPTIONS] [ARGUMENTS]");
System.out.println();
System.out.println("The following actions are available:");
@@ -321,7 +366,7 @@ public class CliFrontendParser {
System.out.println();
}
- public static void printHelpForRun(Collection> customCommandLines) {
+ public static void printHelpForRun(Collection customCommandLines) {
HelpFormatter formatter = new HelpFormatter();
formatter.setLeftPadding(5);
formatter.setWidth(80);
@@ -349,7 +394,7 @@ public class CliFrontendParser {
System.out.println();
}
- public static void printHelpForList(Collection> customCommandLines) {
+ public static void printHelpForList(Collection customCommandLines) {
HelpFormatter formatter = new HelpFormatter();
formatter.setLeftPadding(5);
formatter.setWidth(80);
@@ -364,7 +409,7 @@ public class CliFrontendParser {
System.out.println();
}
- public static void printHelpForStop(Collection> customCommandLines) {
+ public static void printHelpForStop(Collection customCommandLines) {
HelpFormatter formatter = new HelpFormatter();
formatter.setLeftPadding(5);
formatter.setWidth(80);
@@ -379,7 +424,7 @@ public class CliFrontendParser {
System.out.println();
}
- public static void printHelpForCancel(Collection> customCommandLines) {
+ public static void printHelpForCancel(Collection customCommandLines) {
HelpFormatter formatter = new HelpFormatter();
formatter.setLeftPadding(5);
formatter.setWidth(80);
@@ -394,7 +439,7 @@ public class CliFrontendParser {
System.out.println();
}
- public static void printHelpForSavepoint(Collection> customCommandLines) {
+ public static void printHelpForSavepoint(Collection customCommandLines) {
HelpFormatter formatter = new HelpFormatter();
formatter.setLeftPadding(5);
formatter.setWidth(80);
@@ -415,7 +460,7 @@ public class CliFrontendParser {
* @param runOptions True if the run options should be printed, False to print only general options
*/
private static void printCustomCliOptions(
- Collection> customCommandLines,
+ Collection customCommandLines,
HelpFormatter formatter,
boolean runOptions) {
// prints options from all available command-line classes
@@ -445,17 +490,6 @@ public class CliFrontendParser {
// Line Parsing
// --------------------------------------------------------------------------------------------
- public static RunOptions parseRunCommand(String[] args) throws CliArgsException {
- try {
- DefaultParser parser = new DefaultParser();
- CommandLine line = parser.parse(RUN_OPTIONS, args, true);
- return new RunOptions(line);
- }
- catch (ParseException e) {
- throw new CliArgsException(e.getMessage());
- }
- }
-
public static CommandLine parse(Options options, String[] args, boolean stopAtNonOptions) throws CliArgsException {
final DefaultParser parser = new DefaultParser();
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
index e93997490e8dfa86a912640796b971c240a46e80..4241b1c9d51f6f5730ad07c51a5a7901e2d6639b 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
@@ -18,19 +18,16 @@
package org.apache.flink.client.cli;
-import org.apache.flink.client.deployment.ClusterDescriptor;
-import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.FlinkException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
-import javax.annotation.Nullable;
-
/**
* Custom command-line interface to load hooks for the command-line interface.
*/
-public interface CustomCommandLine {
+public interface CustomCommandLine {
/**
* Signals whether the custom command-line wants to execute or not.
@@ -59,37 +56,12 @@ public interface CustomCommandLine {
void addGeneralOptions(Options baseOptions);
/**
- * Create a {@link ClusterDescriptor} from the given configuration, configuration directory
- * and the command line.
- *
- * @param commandLine containing command line options relevant for the ClusterDescriptor
- * @return ClusterDescriptor
- * @throws FlinkException if the ClusterDescriptor could not be created
- */
- ClusterDescriptor createClusterDescriptor(CommandLine commandLine) throws FlinkException;
-
- /**
- * Returns the cluster id if a cluster id was specified on the command line, otherwise it
- * returns null.
- *
- *
A cluster id identifies a running cluster, e.g. the Yarn application id for a Flink
- * cluster running on Yarn.
- *
- * @param commandLine containing command line options relevant for the cluster id retrieval
- * @return Cluster id identifying the cluster to deploy jobs to or null
- */
- @Nullable
- T getClusterId(CommandLine commandLine);
-
- /**
- * Returns the {@link ClusterSpecification} specified by the configuration and the command
- * line options. This specification can be used to deploy a new Flink cluster.
+ * Override configuration settings by specified command line options.
*
- * @param commandLine containing command line options relevant for the ClusterSpecification
- * @return ClusterSpecification for a new Flink cluster
- * @throws FlinkException if the ClusterSpecification could not be created
+ * @param commandLine containing the overriding values
+ * @return the effective configuration with the overridden configuration settings
*/
- ClusterSpecification getClusterSpecification(CommandLine commandLine) throws FlinkException;
+ Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException;
default CommandLine parseCommandLineOptions(String[] args, boolean stopAtNonOptions) throws CliArgsException {
final Options options = new Options();
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java b/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java
index e9ed9af9533c87a68871229e94bce03bc58433d7..12456886fba0eb43ec8227605f942a41e71200c8 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java
@@ -18,21 +18,17 @@
package org.apache.flink.client.cli;
-import org.apache.flink.client.deployment.ClusterSpecification;
-import org.apache.flink.client.deployment.StandaloneClusterDescriptor;
-import org.apache.flink.client.deployment.StandaloneClusterId;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.FlinkException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
-import javax.annotation.Nullable;
-
/**
* The default CLI which is used for interaction with standalone clusters.
*/
-public class DefaultCLI extends AbstractCustomCommandLine {
+public class DefaultCLI extends AbstractCustomCommandLine {
+
+ public static final String ID = "default";
public DefaultCLI(Configuration configuration) {
super(configuration);
@@ -46,30 +42,11 @@ public class DefaultCLI extends AbstractCustomCommandLine {
@Override
public String getId() {
- return "default";
+ return ID;
}
@Override
public void addGeneralOptions(Options baseOptions) {
super.addGeneralOptions(baseOptions);
}
-
- @Override
- public StandaloneClusterDescriptor createClusterDescriptor(
- CommandLine commandLine) throws FlinkException {
- final Configuration effectiveConfiguration = applyCommandLineOptionsToConfiguration(commandLine);
-
- return new StandaloneClusterDescriptor(effectiveConfiguration);
- }
-
- @Override
- @Nullable
- public StandaloneClusterId getClusterId(CommandLine commandLine) {
- return StandaloneClusterId.getInstance();
- }
-
- @Override
- public ClusterSpecification getClusterSpecification(CommandLine commandLine) {
- return new ClusterSpecification.ClusterSpecificationBuilder().createClusterSpecification();
- }
}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java
new file mode 100644
index 0000000000000000000000000000000000000000..b5669f1cd1af08b491144a7e760785bea852e58f
--- /dev/null
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Accessor that exposes config settings that are relevant for execution from an underlying {@link Configuration}.
+ */
+@Internal
+public class ExecutionConfigAccessor {
+
+ private final Configuration configuration;
+
+ private ExecutionConfigAccessor(final Configuration configuration) {
+ this.configuration = checkNotNull(configuration);
+ }
+
+ /**
+ * Creates an {@link ExecutionConfigAccessor} based on the provided {@link Configuration}.
+ */
+ public static ExecutionConfigAccessor fromConfiguration(final Configuration configuration) {
+ return new ExecutionConfigAccessor(checkNotNull(configuration));
+ }
+
+ /**
+ * Creates an {@link ExecutionConfigAccessor} based on the provided {@link ProgramOptions} as provided by the user through the CLI.
+ */
+ public static ExecutionConfigAccessor fromProgramOptions(final ProgramOptions options, final List jobJars) {
+ checkNotNull(options);
+ checkNotNull(jobJars);
+
+ final Configuration configuration = new Configuration();
+
+ if (options.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT) {
+ configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, options.getParallelism());
+ }
+
+ configuration.setBoolean(DeploymentOptions.ATTACHED, !options.getDetachedMode());
+ configuration.setBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED, options.isShutdownOnAttachedExit());
+
+ ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, options.getClasspaths(), URL::toString);
+ ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.JARS, jobJars, URL::toString);
+
+ SavepointRestoreSettings.toConfiguration(options.getSavepointRestoreSettings(), configuration);
+
+ return new ExecutionConfigAccessor(configuration);
+ }
+
+ public Configuration applyToConfiguration(final Configuration baseConfiguration) {
+ baseConfiguration.addAll(configuration);
+ return baseConfiguration;
+ }
+
+ public List getJars() {
+ return decodeUrlList(configuration, PipelineOptions.JARS);
+ }
+
+ public List getClasspaths() {
+ return decodeUrlList(configuration, PipelineOptions.CLASSPATHS);
+ }
+
+ private List decodeUrlList(final Configuration configuration, final ConfigOption> configOption) {
+ return ConfigUtils.decodeListFromConfig(configuration, configOption, url -> {
+ try {
+ return new URL(url);
+ } catch (MalformedURLException e) {
+ throw new IllegalArgumentException("Invalid URL", e);
+ }
+ });
+ }
+
+ public int getParallelism() {
+ return configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
+ }
+
+ public boolean getDetachedMode() {
+ return !configuration.getBoolean(DeploymentOptions.ATTACHED);
+ }
+
+ public SavepointRestoreSettings getSavepointRestoreSettings() {
+ return SavepointRestoreSettings.fromConfiguration(configuration);
+ }
+
+ public boolean isShutdownOnAttachedExit() {
+ return configuration.getBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED);
+ }
+}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java
new file mode 100644
index 0000000000000000000000000000000000000000..4b808a8622772698f1f24714f8aa6007b4c83547
--- /dev/null
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.UnmodifiableConfiguration;
+import org.apache.flink.core.execution.DefaultExecutorServiceLoader;
+import org.apache.flink.core.execution.PipelineExecutor;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A generic implementation of the {@link CustomCommandLine} that only expects
+ * the execution.target parameter to be explicitly specified and simply forwards the
+ * rest of the options specified with -D to the corresponding {@link PipelineExecutor}
+ * for further parsing.
+ */
+@Internal
+public class ExecutorCLI implements CustomCommandLine {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ExecutorCLI.class);
+
+ private static final String ID = "executor";
+
+ private final Option executorOption = new Option("e", "executor", true,
+ "The name of the executor to be used for executing the given job, which is equivalent " +
+ "to the \"" + DeploymentOptions.TARGET.key() + "\" config option. The " +
+ "currently available executors are: " + getExecutorFactoryNames() + ".");
+
+ /**
+ * Dynamic properties allow the user to specify additional configuration values with -D, such as
+ * -Dfs.overwrite-files=true -Dtaskmanager.memory.network.min=536346624.
+ */
+ private final Option dynamicProperties = Option.builder("D")
+ .argName("property=value")
+ .numberOfArgs(2)
+ .valueSeparator('=')
+ .desc("Generic configuration options for execution/deployment and for the configured " +
+ "executor. The available options can be found at " +
+ "https://ci.apache.org/projects/flink/flink-docs-stable/ops/config.html")
+ .build();
+
+ private final Configuration baseConfiguration;
+
+ public ExecutorCLI(final Configuration configuration) {
+ this.baseConfiguration = new UnmodifiableConfiguration(checkNotNull(configuration));
+ }
+
+ @Override
+ public boolean isActive(CommandLine commandLine) {
+ return baseConfiguration.getOptional(DeploymentOptions.TARGET).isPresent()
+ || commandLine.hasOption(executorOption.getOpt());
+ }
+
+ @Override
+ public String getId() {
+ return ID;
+ }
+
+ @Override
+ public void addRunOptions(Options baseOptions) {
+ // nothing to add here
+ }
+
+ @Override
+ public void addGeneralOptions(Options baseOptions) {
+ baseOptions.addOption(executorOption);
+ baseOptions.addOption(dynamicProperties);
+ }
+
+ @Override
+ public Configuration applyCommandLineOptionsToConfiguration(final CommandLine commandLine) {
+ final Configuration effectiveConfiguration = new Configuration(baseConfiguration);
+
+ final String executorName = commandLine.getOptionValue(executorOption.getOpt());
+ if (executorName != null) {
+ effectiveConfiguration.setString(DeploymentOptions.TARGET, executorName);
+ }
+
+ encodeDynamicProperties(commandLine, effectiveConfiguration);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Effective Configuration: {}", effectiveConfiguration);
+ }
+
+ return effectiveConfiguration;
+ }
+
+ private void encodeDynamicProperties(final CommandLine commandLine, final Configuration effectiveConfiguration) {
+ final Properties properties = commandLine.getOptionProperties(dynamicProperties.getOpt());
+ properties.stringPropertyNames()
+ .forEach(key -> {
+ final String value = properties.getProperty(key);
+ if (value != null) {
+ effectiveConfiguration.setString(key, value);
+ } else {
+ effectiveConfiguration.setString(key, "true");
+ }
+ });
+ }
+
+ private static String getExecutorFactoryNames() {
+ return DefaultExecutorServiceLoader.INSTANCE.getExecutorNames()
+ .map(name -> String.format("\"%s\"", name))
+ .collect(Collectors.joining(", "));
+ }
+}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java
index e3a9907f7cf7ce3100450f7ced5c99eb401b7e19..7481ff9d0de1ff42fdc9ab1e6786fb1af95ad6ab 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java
@@ -22,12 +22,15 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import static org.apache.flink.client.cli.CliFrontendParser.ARGS_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.CLASSPATH_OPTION;
@@ -35,8 +38,11 @@ import static org.apache.flink.client.cli.CliFrontendParser.CLASS_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.DETACHED_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.JAR_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.PARALLELISM_OPTION;
+import static org.apache.flink.client.cli.CliFrontendParser.PYARCHIVE_OPTION;
+import static org.apache.flink.client.cli.CliFrontendParser.PYEXEC_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.PYFILES_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.PYMODULE_OPTION;
+import static org.apache.flink.client.cli.CliFrontendParser.PYREQUIREMENTS_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.PY_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.SHUTDOWN_IF_ATTACHED_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.YARN_DETACHED_OPTION;
@@ -44,7 +50,7 @@ import static org.apache.flink.client.cli.CliFrontendParser.YARN_DETACHED_OPTION
/**
* Base class for command line options that refer to a JAR file program.
*/
-public abstract class ProgramOptions extends CommandLineOptions {
+public class ProgramOptions extends CommandLineOptions {
private final String jarFilePath;
@@ -67,7 +73,7 @@ public abstract class ProgramOptions extends CommandLineOptions {
*/
private final boolean isPython;
- protected ProgramOptions(CommandLine line) throws CliArgsException {
+ public ProgramOptions(CommandLine line) throws CliArgsException {
super(line);
String[] args = line.hasOption(ARGS_OPTION.getOpt()) ?
@@ -79,52 +85,24 @@ public abstract class ProgramOptions extends CommandLineOptions {
isPython = line.hasOption(PY_OPTION.getOpt()) | line.hasOption(PYMODULE_OPTION.getOpt())
| "org.apache.flink.client.python.PythonGatewayServer".equals(entryPointClass);
- // If specified the option -py(--python)
- if (line.hasOption(PY_OPTION.getOpt())) {
- // Cannot use option -py and -pym simultaneously.
- if (line.hasOption(PYMODULE_OPTION.getOpt())) {
- throw new CliArgsException("Cannot use option -py and -pym simultaneously.");
- }
- // The cli cmd args which will be transferred to PythonDriver will be transformed as follows:
- // CLI cmd : -py ${python.py} pyfs [optional] ${py-files} [optional] ${other args}.
- // PythonDriver args: py ${python.py} [optional] pyfs [optional] ${py-files} [optional] ${other args}.
- // -------------------------------transformed-------------------------------------------------------
- // e.g. -py wordcount.py(CLI cmd) -----------> py wordcount.py(PythonDriver args)
- // e.g. -py wordcount.py -pyfs file:///AAA.py,hdfs:///BBB.py --input in.txt --output out.txt(CLI cmd)
- // -----> -py wordcount.py -pyfs file:///AAA.py,hdfs:///BBB.py --input in.txt --output out.txt(PythonDriver args)
- String[] newArgs;
- int argIndex;
- if (line.hasOption(PYFILES_OPTION.getOpt())) {
- newArgs = new String[args.length + 4];
- newArgs[2] = "-" + PYFILES_OPTION.getOpt();
- newArgs[3] = line.getOptionValue(PYFILES_OPTION.getOpt());
- argIndex = 4;
- } else {
- newArgs = new String[args.length + 2];
- argIndex = 2;
- }
- newArgs[0] = "-" + PY_OPTION.getOpt();
- newArgs[1] = line.getOptionValue(PY_OPTION.getOpt());
- System.arraycopy(args, 0, newArgs, argIndex, args.length);
- args = newArgs;
- }
-
- // If specified the option -pym(--pyModule)
- if (line.hasOption(PYMODULE_OPTION.getOpt())) {
- // If you specify the option -pym, you should specify the option --pyFiles simultaneously.
- if (!line.hasOption(PYFILES_OPTION.getOpt())) {
- throw new CliArgsException("-pym must be used in conjunction with `--pyFiles`");
+ if (isPython) {
+ // copy python related parameters to program args and place them in front of user parameters
+ List pyArgList = new ArrayList<>();
+ Set
v{{ site.version_title }}
+
+{% top %}
+
+## Tasks and Operator Chains
+
+For distributed execution, Flink *chains* operator subtasks together into
+*tasks*. Each task is executed by one thread. Chaining operators together into
+tasks is a useful optimization: it reduces the overhead of thread-to-thread
+handover and buffering, and increases overall throughput while decreasing
+latency. The chaining behavior can be configured; see the [chaining docs]({{
+site.baseurl }}{% link dev/stream/operators/index.md
+%}#task-chaining-and-resource-groups) for details.
+
+The sample dataflow in the figure below is executed with five subtasks, and
+hence with five parallel threads.
+
+
+
+{% top %}
+
+## Task Slots and Resources
+
+Each worker (TaskManager) is a *JVM process*, and may execute one or more
+subtasks in separate threads. To control how many tasks a worker accepts, a
+worker has so called **task slots** (at least one).
+
+Each *task slot* represents a fixed subset of resources of the TaskManager. A
+TaskManager with three slots, for example, will dedicate 1/3 of its managed
+memory to each slot. Slotting the resources means that a subtask will not
+compete with subtasks from other jobs for managed memory, but instead has a
+certain amount of reserved managed memory. Note that no CPU isolation happens
+here; currently slots only separate the managed memory of tasks.
+
+By adjusting the number of task slots, users can define how subtasks are
+isolated from each other. Having one slot per TaskManager means each task
+group runs in a separate JVM (which can be started in a separate container, for
+example). Having multiple slots means more subtasks share the same JVM. Tasks
+in the same JVM share TCP connections (via multiplexing) and heartbeat
+messages. They may also share data sets and data structures, thus reducing the
+per-task overhead.
+
+
+
+By default, Flink allows subtasks to share slots even if they are subtasks of
+different tasks, so long as they are from the same job. The result is that one
+slot may hold an entire pipeline of the job. Allowing this *slot sharing* has
+two main benefits:
+
+ - A Flink cluster needs exactly as many task slots as the highest parallelism
+ used in the job. No need to calculate how many tasks (with varying
+ parallelism) a program contains in total.
+
+ - It is easier to get better resource utilization. Without slot sharing, the
+ non-intensive *source/map()* subtasks would block as many resources as the
+ resource intensive *window* subtasks. With slot sharing, increasing the
+ base parallelism in our example from two to six yields full utilization of
+ the slotted resources, while making sure that the heavy subtasks are fairly
+ distributed among the TaskManagers.
+
+
+
+{% top %}
diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md
index 22f2c99ef2ef177bdfb2e83874383a5a7a763be3..0645786f8ba0ce2aecbba39a158356ed95ba0797 100644
--- a/docs/concepts/glossary.md
+++ b/docs/concepts/glossary.md
@@ -1,6 +1,6 @@
---
title: Glossary
-nav-pos: 3
+nav-pos: 10
nav-title: Glossary
nav-parent_id: concepts
---
@@ -79,8 +79,8 @@ whole [Flink Master](#flink-master) was called JobManager.
#### Logical Graph
A logical graph is a directed graph describing the high-level logic of a stream processing program.
-The nodes are [Operators](#operator) and the edges indicate input/output-relationships or
-data streams or data sets.
+The nodes are [Operators](#operator) and the edges indicate input/output-relationships of the
+operators and correspond to data streams or data sets.
#### Managed State
@@ -161,6 +161,6 @@ subsequent Tasks.
A Transformation is applied on one or more data streams or data sets and results in one or more
output data streams or data sets. A transformation might change a data stream or data set on a
per-record basis, but might also only change its partitioning or perform an aggregation. While
-[Operators](#operator) and [Functions](#function)) are the "physical" parts of Flink's API,
-Transformations are only an API concept. Specifically, most - but not all - transformations are
+[Operators](#operator) and [Functions](#function) are the "physical" parts of Flink's API,
+Transformations are only an API concept. Specifically, most transformations are
implemented by certain [Operators](#operator).
diff --git a/docs/concepts/glossary.zh.md b/docs/concepts/glossary.zh.md
index 5c5bd1d4cbfe6a0ccc3db4850143ac490ddb24b1..8efd2f101a7ebd365bc35d8735770970b34fcaf1 100644
--- a/docs/concepts/glossary.zh.md
+++ b/docs/concepts/glossary.zh.md
@@ -1,6 +1,6 @@
---
title: 词汇表
-nav-pos: 3
+nav-pos: 10
nav-title: 词汇表
nav-parent_id: concepts
---
diff --git a/docs/concepts/index.md b/docs/concepts/index.md
index 0c59f0462c7961be4d371c4c52718380b56ccf07..b236a5f4c8f9ae016c1e4c16628bababf4fc01d6 100644
--- a/docs/concepts/index.md
+++ b/docs/concepts/index.md
@@ -4,9 +4,9 @@ nav-id: concepts
nav-pos: 2
nav-title: ' Concepts'
nav-parent_id: root
-layout: redirect
-redirect: /concepts/programming-model.html
+nav-show_overview: true
permalink: /concepts/index.html
+always-expand: true
---
+
+Flink offers different levels of abstraction for developing streaming/batch applications.
+
+
+
+ - The lowest level abstraction simply offers **stateful streaming**. It is
+ embedded into the [DataStream API]({{ site.baseurl}}{% link
+ dev/datastream_api.md %}) via the [Process Function]({{ site.baseurl }}{%
+ link dev/stream/operators/process_function.md %}). It allows users freely
+ process events from one or more streams, and use consistent fault tolerant
+ *state*. In addition, users can register event time and processing time
+ callbacks, allowing programs to realize sophisticated computations.
+
+ - In practice, most applications would not need the above described low level
+ abstraction, but would instead program against the **Core APIs** like the
+ [DataStream API]({{ site.baseurl }}{% link dev/datastream_api.md %})
+ (bounded/unbounded streams) and the [DataSet API]({{ site.baseurl }}{% link
+ dev/batch/index.md %}) (bounded data sets). These fluent APIs offer the
+ common building blocks for data processing, like various forms of
+ user-specified transformations, joins, aggregations, windows, state, etc.
+ Data types processed in these APIs are represented as classes in the
+ respective programming languages.
+
+ The low level *Process Function* integrates with the *DataStream API*,
+ making it possible to go the lower level abstraction for certain operations
+ only. The *DataSet API* offers additional primitives on bounded data sets,
+ like loops/iterations.
+
+ - The **Table API** is a declarative DSL centered around *tables*, which may
+ be dynamically changing tables (when representing streams). The [Table
+ API]({{ site.baseurl }}{% link dev/table/index.md %}) follows the
+ (extended) relational model: Tables have a schema attached (similar to
+ tables in relational databases) and the API offers comparable operations,
+ such as select, project, join, group-by, aggregate, etc. Table API
+ programs declaratively define *what logical operation should be done*
+ rather than specifying exactly *how the code for the operation looks*.
+ Though the Table API is extensible by various types of user-defined
+ functions, it is less expressive than the *Core APIs*, but more concise to
+ use (less code to write). In addition, Table API programs also go through
+ an optimizer that applies optimization rules before execution.
+
+ One can seamlessly convert between tables and *DataStream*/*DataSet*,
+ allowing programs to mix *Table API* and with the *DataStream* and
+ *DataSet* APIs.
+
+ - The highest level abstraction offered by Flink is **SQL**. This abstraction
+ is similar to the *Table API* both in semantics and expressiveness, but
+ represents programs as SQL query expressions. The [SQL]({{ site.baseurl
+ }}{% link dev/table/index.md %}#sql) abstraction closely interacts with the
+ Table API, and SQL queries can be executed over tables defined in the
+ *Table API*.
+
+This _concepts_ section explains the basic concepts behind the different APIs,
+that is the concepts behind Flink as a stateful and timely stream processing
+system.
diff --git a/docs/concepts/index.zh.md b/docs/concepts/index.zh.md
index e3dfcc04c3c19ec27708ded5335bdc1dfa443465..2498a434abb5b07de86f134f6dd736bf85118bab 100644
--- a/docs/concepts/index.zh.md
+++ b/docs/concepts/index.zh.md
@@ -4,9 +4,9 @@ nav-id: concepts
nav-pos: 2
nav-title: ' 概念'
nav-parent_id: root
-layout: redirect
-redirect: /concepts/programming-model.html
+nav-show_overview: true
permalink: /concepts/index.html
+always-expand: true
---
+
+Flink offers different levels of abstraction for developing streaming/batch applications.
+
+
-
-Often there is a one-to-one correspondence between the transformations in the programs and the operators
-in the dataflow. Sometimes, however, one transformation may consist of multiple transformation operators.
-
-Sources and sinks are documented in the [streaming connectors](../dev/connectors/index.html) and [batch connectors](../dev/batch/connectors.html) docs.
-Transformations are documented in [DataStream operators]({{ site.baseurl }}/dev/stream/operators/index.html) and [DataSet transformations](../dev/batch/dataset_transformations.html).
-
-{% top %}
-
-## Parallel Dataflows
-
-Programs in Flink are inherently parallel and distributed. During execution, a *stream* has one or more **stream partitions**,
-and each *operator* has one or more **operator subtasks**. The operator subtasks are independent of one another, and execute in different threads
-and possibly on different machines or containers.
-
-The number of operator subtasks is the **parallelism** of that particular operator. The parallelism of a stream
-is always that of its producing operator. Different operators of the same program may have different
-levels of parallelism.
-
-
-
-Streams can transport data between two operators in a *one-to-one* (or *forwarding*) pattern, or in a *redistributing* pattern:
-
- - **One-to-one** streams (for example between the *Source* and the *map()* operators in the figure
- above) preserve the partitioning and ordering of the
- elements. That means that subtask[1] of the *map()* operator will see the same elements in the same order as they
- were produced by subtask[1] of the *Source* operator.
-
- - **Redistributing** streams (as between *map()* and *keyBy/window* above, as well as between
- *keyBy/window* and *Sink*) change the partitioning of streams. Each *operator subtask* sends
- data to different target subtasks, depending on the selected transformation. Examples are
- *keyBy()* (which re-partitions by hashing the key), *broadcast()*, or *rebalance()* (which
- re-partitions randomly). In a *redistributing* exchange the ordering among the elements is
- only preserved within each pair of sending and receiving subtasks (for example, subtask[1]
- of *map()* and subtask[2] of *keyBy/window*). So in this example, the ordering within each key
- is preserved, but the parallelism does introduce non-determinism regarding the order in
- which the aggregated results for different keys arrive at the sink.
-
-Details about configuring and controlling parallelism can be found in the docs on [parallel execution](../dev/parallel.html).
-
-{% top %}
-
-## Windows
-
-Aggregating events (e.g., counts, sums) works differently on streams than in batch processing.
-For example, it is impossible to count all elements in a stream,
-because streams are in general infinite (unbounded). Instead, aggregates on streams (counts, sums, etc),
-are scoped by **windows**, such as *"count over the last 5 minutes"*, or *"sum of the last 100 elements"*.
-
-Windows can be *time driven* (example: every 30 seconds) or *data driven* (example: every 100 elements).
-One typically distinguishes different types of windows, such as *tumbling windows* (no overlap),
-*sliding windows* (with overlap), and *session windows* (punctuated by a gap of inactivity).
-
-
-
-More window examples can be found in this [blog post](https://flink.apache.org/news/2015/12/04/Introducing-windows.html).
-More details are in the [window docs](../dev/stream/operators/windows.html).
-
-{% top %}
-
-## Time
-
-When referring to time in a streaming program (for example to define windows), one can refer to different notions
-of time:
-
- - **Event Time** is the time when an event was created. It is usually described by a timestamp in the events,
- for example attached by the producing sensor, or the producing service. Flink accesses event timestamps
- via [timestamp assigners]({{ site.baseurl }}/dev/event_timestamps_watermarks.html).
-
- - **Ingestion time** is the time when an event enters the Flink dataflow at the source operator.
-
- - **Processing Time** is the local time at each operator that performs a time-based operation.
-
-
-
-More details on how to handle time are in the [event time docs]({{ site.baseurl }}/dev/event_time.html).
-
-{% top %}
-
-## Stateful Operations
-
-While many operations in a dataflow simply look at one individual *event at a time* (for example an event parser),
-some operations remember information across multiple events (for example window operators).
-These operations are called **stateful**.
-
-The state of stateful operations is maintained in what can be thought of as an embedded key/value store.
-The state is partitioned and distributed strictly together with the streams that are read by the
-stateful operators. Hence, access to the key/value state is only possible on *keyed streams*, after a *keyBy()* function,
-and is restricted to the values associated with the current event's key. Aligning the keys of streams and state
-makes sure that all state updates are local operations, guaranteeing consistency without transaction overhead.
-This alignment also allows Flink to redistribute the state and adjust the stream partitioning transparently.
-
-
-
-For more information, see the documentation on [state](../dev/stream/state/index.html).
-
-{% top %}
-
-## Checkpoints for Fault Tolerance
-
-Flink implements fault tolerance using a combination of **stream replay** and **checkpointing**. A
-checkpoint is related to a specific point in each of the input streams along with the corresponding state for each
-of the operators. A streaming dataflow can be resumed from a checkpoint while maintaining consistency *(exactly-once
-processing semantics)* by restoring the state of the operators and replaying the events from the
-point of the checkpoint.
-
-The checkpoint interval is a means of trading off the overhead of fault tolerance during execution with the recovery time (the number
-of events that need to be replayed).
-
-The description of the [fault tolerance internals]({{ site.baseurl }}/internals/stream_checkpointing.html) provides
-more information about how Flink manages checkpoints and related topics.
-Details about enabling and configuring checkpointing are in the [checkpointing API docs](../dev/stream/state/checkpointing.html).
-
-
-{% top %}
-
-## Batch on Streaming
-
-Flink executes [batch programs](../dev/batch/index.html) as a special case of streaming programs, where the streams are bounded (finite number of elements).
-A *DataSet* is treated internally as a stream of data. The concepts above thus apply to batch programs in the
-same way as well as they apply to streaming programs, with minor exceptions:
-
- - [Fault tolerance for batch programs](../dev/batch/fault_tolerance.html) does not use checkpointing.
- Recovery happens by fully replaying the streams.
- That is possible, because inputs are bounded. This pushes the cost more towards the recovery,
- but makes the regular processing cheaper, because it avoids checkpoints.
-
- - Stateful operations in the DataSet API use simplified in-memory/out-of-core data structures, rather than
- key/value indexes.
-
- - The DataSet API introduces special synchronized (superstep-based) iterations, which are only possible on
- bounded streams. For details, check out the [iteration docs]({{ site.baseurl }}/dev/batch/iterations.html).
-
-{% top %}
-
-## Next Steps
-
-Continue with the basic concepts in Flink's [Distributed Runtime](runtime.html).
diff --git a/docs/concepts/programming-model.zh.md b/docs/concepts/programming-model.zh.md
deleted file mode 100644
index 99253da5e9bfacc929f127e5b1a6b764bc50f309..0000000000000000000000000000000000000000
--- a/docs/concepts/programming-model.zh.md
+++ /dev/null
@@ -1,217 +0,0 @@
----
-title: 数据流编程模型
-nav-id: programming-model
-nav-pos: 1
-nav-title: 编程模型
-nav-parent_id: concepts
----
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## Levels of Abstraction
-
-Flink offers different levels of abstraction to develop streaming/batch applications.
-
-
-
-{% top %}
-
-## Savepoints
-
-Programs written in the Data Stream API can resume execution from a **savepoint**. Savepoints allow both updating your programs and your Flink cluster without losing any state.
-
-[Savepoints](../ops/state/savepoints.html) are **manually triggered checkpoints**, which take a snapshot of the program and write it out to a state backend. They rely on the regular checkpointing mechanism for this. During execution programs are periodically snapshotted on the worker nodes and produce checkpoints. For recovery only the last completed checkpoint is needed and older checkpoints can be safely discarded as soon as a new one is completed.
-
-Savepoints are similar to these periodic checkpoints except that they are **triggered by the user** and **don't automatically expire** when newer checkpoints are completed. Savepoints can be created from the [command line](../ops/cli.html#savepoints) or when cancelling a job via the [REST API](../monitoring/rest_api.html#cancel-job-with-savepoint).
-
-{% top %}
diff --git a/docs/concepts/runtime.zh.md b/docs/concepts/runtime.zh.md
deleted file mode 100644
index 307c883730435b5dd8f092b1978590a50696269e..0000000000000000000000000000000000000000
--- a/docs/concepts/runtime.zh.md
+++ /dev/null
@@ -1,99 +0,0 @@
----
-title: 分布式运行时环境
-nav-pos: 2
-nav-title: 分布式运行时
-nav-parent_id: concepts
----
-
-
-* This will be replaced by the TOC
-{:toc}
-
-## 任务和算子链
-
-分布式计算中，Flink 将算子（operator）的 subtask *链接（chain）*成 task。每个 task 由一个线程执行。把算子链接成 tasks 能够减少线程间切换和缓冲的开销，在降低延迟的同时提高了整体吞吐量。链接操作的配置详情可参考：[chaining docs](../dev/stream/operators/#task-chaining-and-resource-groups)
-
-下图的 dataflow 由五个 subtasks 执行，因此具有五个并行线程。
-
-
+
+
+
+
+{% top %}
+
+## Event Time and Watermarks
+
+*Note: Flink implements many techniques from the Dataflow Model. For a good
+introduction to event time and watermarks, have a look at the articles below.*
+
+ - [Streaming
+ 101](https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101) by
+ Tyler Akidau
+ - The [Dataflow Model
+ paper](https://research.google.com/pubs/archive/43864.pdf)
+
+
+A stream processor that supports *event time* needs a way to measure the
+progress of event time. For example, a window operator that builds hourly
+windows needs to be notified when event time has passed beyond the end of an
+hour, so that the operator can close the window in progress.
+
+*Event time* can progress independently of *processing time* (measured by wall
+clocks). For example, in one program the current *event time* of an operator
+may trail slightly behind the *processing time* (accounting for a delay in
+receiving the events), while both proceed at the same speed. On the other
+hand, another streaming program might progress through weeks of event time with
+only a few seconds of processing, by fast-forwarding through some historic data
+already buffered in a Kafka topic (or another message queue).
+
+------
+
+The mechanism in Flink to measure progress in event time is **watermarks**.
+Watermarks flow as part of the data stream and carry a timestamp *t*. A
+*Watermark(t)* declares that event time has reached time *t* in that stream,
+meaning that there should be no more elements from the stream with a timestamp
+*t' <= t* (i.e. events with timestamps older or equal to the watermark).
+
+The figure below shows a stream of events with (logical) timestamps, and
+watermarks flowing inline. In this example the events are in order (with
+respect to their timestamps), meaning that the watermarks are simply periodic
+markers in the stream.
+
+
+
+Watermarks are crucial for *out-of-order* streams, as illustrated below, where
+the events are not ordered by their timestamps. In general a watermark is a
+declaration that by that point in the stream, all events up to a certain
+timestamp should have arrived. Once a watermark reaches an operator, the
+operator can advance its internal *event time clock* to the value of the
+watermark.
+
+
+
+Note that event time is inherited by a freshly created stream element (or
+elements) from either the event that produced them or from watermark that
+triggered creation of those elements.
+
+### Watermarks in Parallel Streams
+
+Watermarks are generated at, or directly after, source functions. Each parallel
+subtask of a source function usually generates its watermarks independently.
+These watermarks define the event time at that particular parallel source.
+
+As the watermarks flow through the streaming program, they advance the event
+time at the operators where they arrive. Whenever an operator advances its
+event time, it generates a new watermark downstream for its successor
+operators.
+
+Some operators consume multiple input streams; a union, for example, or
+operators following a *keyBy(...)* or *partition(...)* function. Such an
+operator's current event time is the minimum of its input streams' event times.
+As its input streams update their event times, so does the operator.
+
+The figure below shows an example of events and watermarks flowing through
+parallel streams, and operators tracking event time.
+
+
+
+## Lateness
+
+It is possible that certain elements will violate the watermark condition,
+meaning that even after the *Watermark(t)* has occurred, more elements with
+timestamp *t' <= t* will occur. In fact, in many real world setups, certain
+elements can be arbitrarily delayed, making it impossible to specify a time by
+which all elements of a certain event timestamp will have occurred.
+Furthermore, even if the lateness can be bounded, delaying the watermarks by
+too much is often not desirable, because it causes too much delay in the
+evaluation of event time windows.
+
+For this reason, streaming programs may explicitly expect some *late* elements.
+Late elements are elements that arrive after the system's event time clock (as
+signaled by the watermarks) has already passed the time of the late element's
+timestamp. See [Allowed Lateness]({{ site.baseurl }}{% link
+dev/stream/operators/windows.md %}#allowed-lateness) for more information on
+how to work with late elements in event time windows.
+
+## Windowing
+
+Aggregating events (e.g., counts, sums) works differently on streams than in
+batch processing. For example, it is impossible to count all elements in a
+stream, because streams are in general infinite (unbounded). Instead,
+aggregates on streams (counts, sums, etc), are scoped by **windows**, such as
+*"count over the last 5 minutes"*, or *"sum of the last 100 elements"*.
+
+Windows can be *time driven* (example: every 30 seconds) or *data driven*
+(example: every 100 elements). One typically distinguishes different types of
+windows, such as *tumbling windows* (no overlap), *sliding windows* (with
+overlap), and *session windows* (punctuated by a gap of inactivity).
+
+
-### Bucket Assignment
-
-The bucketing logic defines how the data will be structured into subdirectories inside the base output directory.
-
-Both row and bulk formats use the [DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) as the default assigner.
-By default the DateTimeBucketAssigner creates hourly buckets based on the system default timezone
-with the following format: `yyyy-MM-dd--HH`. Both the date format (i.e. bucket size) and timezone can be
-configured manually.
-
-We can specify a custom [BucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssigner.html) by calling `.withBucketAssigner(assigner)` on the format builders.
-
-Flink comes with two built in BucketAssigners:
-
- - [DateTimeBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.html) : Default time based assigner
- - [BasePathBucketAssigner]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/BasePathBucketAssigner.html) : Assigner that stores all part files in the base path (single global bucket)
-
-### Rolling Policy
-
-The [RollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicy.html) defines when a given in-progress part file will be closed and moved to the pending and later to finished state.
-In combination with the checkpointing interval (pending files become finished on the next checkpoint) this controls how quickly
-part files become available for downstream readers and also the size and number of these parts.
-
-Flink comes with two built-in RollingPolicies:
-
- - [DefaultRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/DefaultRollingPolicy.html)
- - [OnCheckpointRollingPolicy]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/OnCheckpointRollingPolicy.html)
-
-### Part file lifecycle
-
-In order to use the output of the StreamingFileSink in downstream systems, we need to understand the naming and lifecycle of the output files produced.
-
-Part files can be in one of three states:
- 1. **In-progress** : The part file that is currently being written to is in-progress
- 2. **Pending** : Once a part file is closed for writing it becomes pending
- 3. **Finished** : On successful checkpoints pending files become finished
-
-Only finished files are safe to read by downstream systems as those are guaranteed to not be modified later. Finished files can be distinguished by their naming scheme only.
-
-File naming schemes:
- - **In-progress / Pending**: `part-subtaskIndex-partFileIndex.inprogress.uid`
- - **Finished:** `part-subtaskIndex-partFileIndex`
-
-Part file indexes are strictly increasing for any given subtask (in the order they were created). However these indexes are not always sequential. When the job restarts, the next part index for all subtask will be the `max part index + 1`.
-
-Each writer subtask will have a single in-progress part file at any given time for every active bucket, but there can be several pending and finished files.
-
-**Part file example**
-
-To better understand the lifecycle of these files let's look at a simple example with 2 sink subtasks:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- └── part-1-0.inprogress.ea65a428-a1d0-4a0b-bbc5-7a436a75e575
-```
-
-When the part file `part-1-0` is rolled (let's say it becomes too large), it becomes pending but it is not renamed. The sink then opens a new part file: `part-1-1`:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- ├── part-1-0.inprogress.ea65a428-a1d0-4a0b-bbc5-7a436a75e575
- └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
-```
-
-As `part-1-0` is now pending completion, after the next successful checkpoint, it is finalized:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- ├── part-1-0
- └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
-```
-
-New buckets are created as dictated by the bucketing policy, and this doesn't affect currently in-progress files:
-
-```
-└── 2019-08-25--12
- ├── part-0-0.inprogress.bd053eb0-5ecf-4c85-8433-9eff486ac334
- ├── part-1-0
- └── part-1-1.inprogress.bc279efe-b16f-47d8-b828-00ef6e2fbd11
-└── 2019-08-25--13
- └── part-0-2.inprogress.2b475fec-1482-4dea-9946-eb4353b475f1
-```
-
-Old buckets can still receive new records as the bucketing policy is evaluated on a per-record basis.
## File Formats
@@ -222,10 +136,16 @@ specifying an `Encoder` we have to specify [BulkWriter.Factory]({{ site.javadocs
The `BulkWriter` logic defines how new elements added, flushed and how the bulk of records
are finalized for further encoding purposes.
-Flink comes with two built-in BulkWriter factories:
+Flink comes with three built-in BulkWriter factories:
- [ParquetWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/parquet/ParquetWriterFactory.html)
- [SequenceFileWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.html)
+ - [CompressWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/compress/CompressWriterFactory.html)
+
+
+
+For information about how to use time in Flink programs refer to
+[windowing]({{site.baseurl}}{% link dev/stream/operators/windows.md %}) and
+[ProcessFunction]({{ site.baseurl }}{% link
+dev/stream/operators/process_function.md %}).
+* toc
+{:toc}
-### Setting a Time Characteristic
+## Setting a Time Characteristic
The first part of a Flink DataStream program usually sets the base *time characteristic*. That setting
defines how data stream sources behave (for example, whether they will assign timestamps), and what notion of
@@ -103,7 +57,7 @@ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
// env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
-DataStream
+
+
-
-
+
-If the samples show that a task Thread is stuck in a certain internal method call (requesting buffers from the network stack), this indicates that there is back pressure for the task.
+Internally, back pressure is judged based on the availability of output buffers. If there is no available buffer (at least one) for output, then it indicates that there is back pressure for the task.
-By default, the job manager triggers 100 stack traces every 50ms for each task in order to determine back pressure. The ratio you see in the web interface tells you how many of these stack traces were stuck in the internal method call, e.g. `0.01` indicates that only 1 in 100 was stuck in that method.
+By default, the job manager triggers 100 samples every 50ms for each task in order to determine back pressure. The ratio you see in the web interface tells you how many of these samples were indicating back pressure, e.g. `0.01` indicates that only 1 in 100 was back pressured.
- **OK**: 0 <= Ratio <= 0.10
- **LOW**: 0.10 < Ratio <= 0.5
- **HIGH**: 0.5 < Ratio <= 1
-In order to not overload the task managers with stack trace samples, the web interface refreshes samples only after 60 seconds.
+In order to not overload the task managers with back pressure samples, the web interface refreshes samples only after 60 seconds.
## Configuration
You can configure the number of samples for the job manager with the following configuration keys:
- `web.backpressure.refresh-interval`: Time after which available stats are deprecated and need to be refreshed (DEFAULT: 60000, 1 min).
-- `web.backpressure.num-samples`: Number of stack trace samples to take to determine back pressure (DEFAULT: 100).
-- `web.backpressure.delay-between-samples`: Delay between stack trace samples to determine back pressure (DEFAULT: 50, 50 ms).
+- `web.backpressure.num-samples`: Number of samples to take to determine back pressure (DEFAULT: 100).
+- `web.backpressure.delay-between-samples`: Delay between samples to determine back pressure (DEFAULT: 50, 50 ms).
## Example
@@ -66,7 +66,7 @@ You can find the *Back Pressure* tab next to the job overview.
### Sampling In Progress
-This means that the JobManager triggered a stack trace sample of the running tasks. With the default configuration, this takes about 5 seconds to complete.
+This means that the JobManager triggered a back pressure sample of the running tasks. With the default configuration, this takes about 5 seconds to complete.
Note that clicking the row, you trigger the sample for all subtasks of this operator.
diff --git a/docs/monitoring/back_pressure.zh.md b/docs/monitoring/back_pressure.zh.md
index 127f7ffd1af56811979b7048eaa890c68c568d50..4dd171d3644e40f4e13f7d362e9b7998c9597127 100644
--- a/docs/monitoring/back_pressure.zh.md
+++ b/docs/monitoring/back_pressure.zh.md
@@ -36,31 +36,31 @@ Flink Web 界面提供了一个选项卡来监控正在运行 Job 的反压行
`Sink` 正在向上游的 `Source` 算子产生反压。
-## 采样线程
+## 反压采样
-通过不断对每个 Task 的 stack trace 采样来进行反压监控。JobManager 会触发对 Task `Thread.getStackTrace()` 的重复调用。
+通过不断对每个 Task 的反压状态采样来进行反压监控。JobManager 会触发对 Task `Task.isBackPressured()` 的重复调用。
+----
+----
-### Configuration Runtime Environment Variables
-You have to set config with prefix `containerized.master.env.` and `containerized.taskmanager.env.` in order to set redefined environment variable in ApplicationMaster and TaskManager.
+# JVM and Logging Options
-- `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
- For example for passing LD_LIBRARY_PATH as an env variable to the AppMaster, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
+{% include generated/environment_configuration.html %}
+
+# Forwarding Environment Variables
+
+You can configure environment variables to be set on the Flink Master and TaskManager processes started on Yarn/Mesos.
+
+ - `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
+ For example for passing LD_LIBRARY_PATH as an env variable to the Master, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
in the flink-conf.yaml.
-- `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
+
+ - `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
+
+----
+----
+
+# Deprecated Options
+
+These options relate to parts of Flink that are not actively developed any more.
+These options may be removed in a future release.
+
+**DataSet API Optimizer**
+
+{% include generated/optimizer_configuration.html %}
+
+**DataSet API Runtime Algorithms**
+
+{% include generated/algorithm_configuration.html %}
+
+**DataSet File Sinks**
+
+{% include generated/deprecated_file_sinks_section.html %}
+
+----
+----
+
+# Backup
+
+#### Execution
+
+{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
+{% include generated/execution_configuration.html %}
+
+#### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+{% include generated/stream_pipeline_configuration.html %}
+
+#### Checkpointing
+
+{% include generated/execution_checkpointing_configuration.html %}
{% top %}
diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md
index cfbd9cc7c141397defe62ae5751538df5b112fb0..23b7ce1dd4057dfe563018f49d7b92ba2e3b5fe2 100644
--- a/docs/ops/config.zh.md
+++ b/docs/ops/config.zh.md
@@ -23,307 +23,415 @@ specific language governing permissions and limitations
under the License.
-->
-**For single-node setups Flink is ready to go out of the box and you don't need to change the default configuration to get started.**
+All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+
+The configuration is parsed and evaluated when the Flink processes are started. Changes to the configuration file require restarting the relevant processes.
The out of the box configuration will use your default Java installation. You can manually set the environment variable `JAVA_HOME` or the configuration key `env.java.home` in `conf/flink-conf.yaml` if you want to manually override the Java runtime to use.
-This page lists the most common options that are typically needed to set up a well performing (distributed) installation. In addition a full list of all available configuration parameters is listed here.
+* This will be replaced by the TOC
+{:toc}
-All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+# Basic Setup
-The system and run scripts parse the config at startup time. Changes to the configuration file require restarting the Flink JobManager and TaskManagers.
+The default configuration supports starting a single-node Flink session cluster without any changes.
+The options in this section are the ones most commonly needed for a basic distributed Flink setup.
-The configuration files for the TaskManagers can be different, Flink does not assume uniform machines in the cluster.
+**Hostnames / Ports**
-* This will be replaced by the TOC
-{:toc}
+These options are only necessary for *standalone* application- or session deployments ([simple standalone]({{site.baseurl}}/ops/deployment/cluster_setup.html) or [Kubernetes]({{site.baseurl}}/ops/deployment/kubernetes.html)).
-## Common Options
+If you use Flink with [Yarn]({{site.baseurl}}/ops/deployment/yarn_setup.html), [Mesos]({{site.baseurl}}/ops/deployment/mesos.html), or the [*active* Kubernetes integration]({{site.baseurl}}/ops/deployment/native_kubernetes.html), the hostnames and ports are automatically discovered.
-{% include generated/common_section.html %}
+ - `rest.address`, `rest.port`: These are used by the client to connect to Flink. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes) service in front of the Flink Master's REST interface.
-## Full Reference
+ - The `jobmanager.rpc.address` (defaults to *"localhost"*) and `jobmanager.rpc.port` (defaults to *6123*) config entries are used by the TaskManager to connect to the JobManager/ResourceManager. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes internal) service for the Flink master (JobManager). This option is ignored on [setups with high-availability]({{site.baseurl}}/ops/jobmanager_high_availability.html) where the leader election mechanism is used to discover this automatically.
-### HDFS
+**Memory Sizes**
-
+
+When creating a Flink Kubernetes session cluster, the Flink client will first connect to the Kubernetes ApiServer to submit the cluster description, including ConfigMap spec, Job Manager Service spec, Job Manager Deployment spec and Owner Reference.
+Kubernetes will then create the Flink master deployment, during which time the Kubelet will pull the image, prepare and mount the volume, and then execute the start command.
+After the master pod has launched, the Dispatcher and KubernetesResourceManager are available and the cluster is ready to accept one or more jobs.
+
+When users submit jobs through the Flink client, the job graph will be generated by the client and uploaded along with users jars to the Dispatcher.
+A JobMaster for that Job will be then be spawned.
+
+The JobMaster requests resources, known as slots, from the KubernetesResourceManager.
+If no slots are available, the resource manager will bring up TaskManager pods and registering them with the cluster.
+
+{% top %}
diff --git a/docs/ops/deployment/native_kubernetes.zh.md b/docs/ops/deployment/native_kubernetes.zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..10d47e02a7b84abfb84367013c789521649938cf
--- /dev/null
+++ b/docs/ops/deployment/native_kubernetes.zh.md
@@ -0,0 +1,206 @@
+---
+title: "原生 Kubernetes 设置"
+nav-title: Native Kubernetes
+nav-parent_id: deployment
+is_beta: true
+nav-pos: 7
+---
+
+
+This page describes how to deploy a Flink session cluster natively on [Kubernetes](https://kubernetes.io).
+
+* This will be replaced by the TOC
+{:toc}
+
+
+
+