commit e60c777b104e9c6905ecb2b1c040b6b36effdc51
Author: Simon Watson <spw01@protonmail.com>
Date: Fri Jan 14 11:03:21 2022 -0500
Add ES post
diff --git a/2021-emacs-windows.pretty.html b/2021-emacs-windows.pretty.html
index 1786c02..eeb48c0 100644
--- a/2021-emacs-windows.pretty.html
+++ b/2021-emacs-windows.pretty.html
@@ -10,7 +10,7 @@
<meta name="generator" content="Org mode" />
<meta name="author" content="Simon Watson" />
<style type="text/css">body{margin:40px
-auto;max-width:700px;line-height:1.6;font-size:18px;color:#444;padding:0
+auto;max-width:900px;line-height:1.6;font-size:18px;color:#444;padding:0
10px}h1,h2,h3{line-height:1.2}</style>
<script type="text/javascript">
// @license magnet:?xt=urn:btih:e95b018ef3580986a04669f1b5879592219e2a7a&dn=public-domain.txt Public Domain
diff --git a/2022-01-13-es-json.html b/2022-01-13-es-json.html
new file mode 100644
index 0000000..7ee03fd
--- /dev/null
+++ b/2022-01-13-es-json.html
@@ -0,0 +1,317 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+<!-- 2022-01-14 Fri 10:55 -->
+<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title></title>
+<meta name="generator" content="Org mode" />
+<style type="text/css">
+ <!--/*--><![CDATA[/*><!--*/
+ .title { text-align: center;
+ margin-bottom: .2em; }
+ .subtitle { text-align: center;
+ font-size: medium;
+ font-weight: bold;
+ margin-top:0; }
+ .todo { font-family: monospace; color: red; }
+ .done { font-family: monospace; color: green; }
+ .priority { font-family: monospace; color: orange; }
+ .tag { background-color: #eee; font-family: monospace;
+ padding: 2px; font-size: 80%; font-weight: normal; }
+ .timestamp { color: #bebebe; }
+ .timestamp-kwd { color: #5f9ea0; }
+ .org-right { margin-left: auto; margin-right: 0px; text-align: right; }
+ .org-left { margin-left: 0px; margin-right: auto; text-align: left; }
+ .org-center { margin-left: auto; margin-right: auto; text-align: center; }
+ .underline { text-decoration: underline; }
+ #postamble p, #preamble p { font-size: 90%; margin: .2em; }
+ p.verse { margin-left: 3%; }
+ pre {
+ border: 1px solid #ccc;
+ box-shadow: 3px 3px 3px #eee;
+ padding: 8pt;
+ font-family: monospace;
+ overflow: auto;
+ margin: 1.2em;
+ }
+ pre.src {
+ position: relative;
+ overflow: auto;
+ padding-top: 1.2em;
+ }
+ pre.src:before {
+ display: none;
+ position: absolute;
+ background-color: white;
+ top: -10px;
+ right: 10px;
+ padding: 3px;
+ border: 1px solid black;
+ }
+ pre.src:hover:before { display: inline; margin-top: 14px;}
+ /* Languages per Org manual */
+ pre.src-asymptote:before { content: 'Asymptote'; }
+ pre.src-awk:before { content: 'Awk'; }
+ pre.src-C:before { content: 'C'; }
+ /* pre.src-C++ doesn't work in CSS */
+ pre.src-clojure:before { content: 'Clojure'; }
+ pre.src-css:before { content: 'CSS'; }
+ pre.src-D:before { content: 'D'; }
+ pre.src-ditaa:before { content: 'ditaa'; }
+ pre.src-dot:before { content: 'Graphviz'; }
+ pre.src-calc:before { content: 'Emacs Calc'; }
+ pre.src-emacs-lisp:before { content: 'Emacs Lisp'; }
+ pre.src-fortran:before { content: 'Fortran'; }
+ pre.src-gnuplot:before { content: 'gnuplot'; }
+ pre.src-haskell:before { content: 'Haskell'; }
+ pre.src-hledger:before { content: 'hledger'; }
+ pre.src-java:before { content: 'Java'; }
+ pre.src-js:before { content: 'Javascript'; }
+ pre.src-latex:before { content: 'LaTeX'; }
+ pre.src-ledger:before { content: 'Ledger'; }
+ pre.src-lisp:before { content: 'Lisp'; }
+ pre.src-lilypond:before { content: 'Lilypond'; }
+ pre.src-lua:before { content: 'Lua'; }
+ pre.src-matlab:before { content: 'MATLAB'; }
+ pre.src-mscgen:before { content: 'Mscgen'; }
+ pre.src-ocaml:before { content: 'Objective Caml'; }
+ pre.src-octave:before { content: 'Octave'; }
+ pre.src-org:before { content: 'Org mode'; }
+ pre.src-oz:before { content: 'OZ'; }
+ pre.src-plantuml:before { content: 'Plantuml'; }
+ pre.src-processing:before { content: 'Processing.js'; }
+ pre.src-python:before { content: 'Python'; }
+ pre.src-R:before { content: 'R'; }
+ pre.src-ruby:before { content: 'Ruby'; }
+ pre.src-sass:before { content: 'Sass'; }
+ pre.src-scheme:before { content: 'Scheme'; }
+ pre.src-screen:before { content: 'Gnu Screen'; }
+ pre.src-sed:before { content: 'Sed'; }
+ pre.src-sh:before { content: 'shell'; }
+ pre.src-sql:before { content: 'SQL'; }
+ pre.src-sqlite:before { content: 'SQLite'; }
+ /* additional languages in org.el's org-babel-load-languages alist */
+ pre.src-forth:before { content: 'Forth'; }
+ pre.src-io:before { content: 'IO'; }
+ pre.src-J:before { content: 'J'; }
+ pre.src-makefile:before { content: 'Makefile'; }
+ pre.src-maxima:before { content: 'Maxima'; }
+ pre.src-perl:before { content: 'Perl'; }
+ pre.src-picolisp:before { content: 'Pico Lisp'; }
+ pre.src-scala:before { content: 'Scala'; }
+ pre.src-shell:before { content: 'Shell Script'; }
+ pre.src-ebnf2ps:before { content: 'ebfn2ps'; }
+ /* additional language identifiers per "defun org-babel-execute"
+ in ob-*.el */
+ pre.src-cpp:before { content: 'C++'; }
+ pre.src-abc:before { content: 'ABC'; }
+ pre.src-coq:before { content: 'Coq'; }
+ pre.src-groovy:before { content: 'Groovy'; }
+ /* additional language identifiers from org-babel-shell-names in
+ ob-shell.el: ob-shell is the only babel language using a lambda to put
+ the execution function name together. */
+ pre.src-bash:before { content: 'bash'; }
+ pre.src-csh:before { content: 'csh'; }
+ pre.src-ash:before { content: 'ash'; }
+ pre.src-dash:before { content: 'dash'; }
+ pre.src-ksh:before { content: 'ksh'; }
+ pre.src-mksh:before { content: 'mksh'; }
+ pre.src-posh:before { content: 'posh'; }
+ /* Additional Emacs modes also supported by the LaTeX listings package */
+ pre.src-ada:before { content: 'Ada'; }
+ pre.src-asm:before { content: 'Assembler'; }
+ pre.src-caml:before { content: 'Caml'; }
+ pre.src-delphi:before { content: 'Delphi'; }
+ pre.src-html:before { content: 'HTML'; }
+ pre.src-idl:before { content: 'IDL'; }
+ pre.src-mercury:before { content: 'Mercury'; }
+ pre.src-metapost:before { content: 'MetaPost'; }
+ pre.src-modula-2:before { content: 'Modula-2'; }
+ pre.src-pascal:before { content: 'Pascal'; }
+ pre.src-ps:before { content: 'PostScript'; }
+ pre.src-prolog:before { content: 'Prolog'; }
+ pre.src-simula:before { content: 'Simula'; }
+ pre.src-tcl:before { content: 'tcl'; }
+ pre.src-tex:before { content: 'TeX'; }
+ pre.src-plain-tex:before { content: 'Plain TeX'; }
+ pre.src-verilog:before { content: 'Verilog'; }
+ pre.src-vhdl:before { content: 'VHDL'; }
+ pre.src-xml:before { content: 'XML'; }
+ pre.src-nxml:before { content: 'XML'; }
+ /* add a generic configuration mode; LaTeX export needs an additional
+ (add-to-list 'org-latex-listings-langs '(conf " ")) in .emacs */
+ pre.src-conf:before { content: 'Configuration File'; }
+
+ table { border-collapse:collapse; }
+ caption.t-above { caption-side: top; }
+ caption.t-bottom { caption-side: bottom; }
+ td, th { vertical-align:top; }
+ th.org-right { text-align: center; }
+ th.org-left { text-align: center; }
+ th.org-center { text-align: center; }
+ td.org-right { text-align: right; }
+ td.org-left { text-align: left; }
+ td.org-center { text-align: center; }
+ dt { font-weight: bold; }
+ .footpara { display: inline; }
+ .footdef { margin-bottom: 1em; }
+ .figure { padding: 1em; }
+ .figure p { text-align: center; }
+ .equation-container {
+ display: table;
+ text-align: center;
+ width: 100%;
+ }
+ .equation {
+ vertical-align: middle;
+ }
+ .equation-label {
+ display: table-cell;
+ text-align: right;
+ vertical-align: middle;
+ }
+ .inlinetask {
+ padding: 10px;
+ border: 2px solid gray;
+ margin: 10px;
+ background: #ffffcc;
+ }
+ #org-div-home-and-up
+ { text-align: right; font-size: 70%; white-space: nowrap; }
+ textarea { overflow-x: auto; }
+ .linenr { font-size: smaller }
+ .code-highlighted { background-color: #ffff00; }
+ .org-info-js_info-navigation { border-style: none; }
+ #org-info-js_console-label
+ { font-size: 10px; font-weight: bold; white-space: nowrap; }
+ .org-info-js_search-highlight
+ { background-color: #ffff00; color: #000000; font-weight: bold; }
+ .org-svg { width: 90%; }
+ /*]]>*/-->
+</style>
+<script type="text/javascript">
+// @license magnet:?xt=urn:btih:e95b018ef3580986a04669f1b5879592219e2a7a&dn=public-domain.txt Public Domain
+<!--/*--><![CDATA[/*><!--*/
+ function CodeHighlightOn(elem, id)
+ {
+ var target = document.getElementById(id);
+ if(null != target) {
+ elem.classList.add("code-highlighted");
+ target.classList.add("code-highlighted");
+ }
+ }
+ function CodeHighlightOff(elem, id)
+ {
+ var target = document.getElementById(id);
+ if(null != target) {
+ elem.classList.remove("code-highlighted");
+ target.classList.remove("code-highlighted");
+ }
+ }
+ /*]]>*///-->
+// @license-end
+</script>
+</head>
+<body>
+<div id="content">
+<div id="table-of-contents">
+<h2>Table of Contents</h2>
+<div id="text-table-of-contents">
+<ul>
+<li><a href="#org4003f02">1. How to parse single line JSON logs for ElasticSearch Cloud</a>
+<ul>
+<li><a href="#org60dc23d">1.1. The Problem</a></li>
+<li><a href="#orgdfc5189">1.2. The Solution</a></li>
+</ul>
+</li>
+</ul>
+</div>
+</div>
+<p>
+-<b>- org-mode -</b>-
+</p>
+
+<div id="outline-container-org4003f02" class="outline-2">
+<h2 id="org4003f02"><span class="section-number-2">1</span> How to parse single line JSON logs for ElasticSearch Cloud</h2>
+<div class="outline-text-2" id="text-1">
+<p>
+This is going to be a brief blog post, but wanted to jot down a few things as solving this "easy" issue has taken me the better part of 4 hours.
+</p>
+</div>
+<div id="outline-container-org60dc23d" class="outline-3">
+<h3 id="org60dc23d"><span class="section-number-3">1.1</span> The Problem</h3>
+<div class="outline-text-3" id="text-1-1">
+<p>
+I have a number of weirdly formatted logs that developers would like to be able to easily search through and get insights from. The developers control this log format,
+but its an embedded environment and it's "non-trivial" to modify the format. I wrote a Perl script that will read in these developer logs and regex out
+key fields I'm interested in, transforming them like so (fake data):
+</p>
+
+<div class="org-src-container">
+<pre class="src src-shell"># Original log line
+# LOG LEVEL # DATE & TIME # FUNCTION NAME/LINE NUMBER # LOG MESSAGE
+[DEBUG] 2020/9/10 - 13:59:23 | some_function_name 166: some log message
+
+# PARSED LOG LINE
+{"log_level":"Debug","timestamp":"2020-09-10T13:59:23","function_name":"some_function_name","line_number":"166","message":"some log message"}
+</pre>
+</div>
+
+<p>
+After setting up this log parser and filebeat, I started processing these logs into a hosted ElasticSearch cloud instance. To my surprised, the JSON fields were
+not indexed, meaning I couldn't perform KQL searches like <code>timestamp:2020-09*</code> to get all log lines from that month.
+</p>
+</div>
+</div>
+
+<div id="outline-container-orgdfc5189" class="outline-3">
+<h3 id="orgdfc5189"><span class="section-number-3">1.2</span> The Solution</h3>
+<div class="outline-text-3" id="text-1-2">
+<p>
+To Elastic's credit, it's actually incredibly simple to get this behavior with filebeat, all I needed to do was add the following to the <code>/etc/filebeat/filebeat.yml</code>
+file under the <code>processors</code> field (This is on filebeat versions 7.x):
+</p>
+
+<div class="org-src-container">
+<pre class="src src-yaml">processors:
+ - decode_json_fields:
+ fields: ["line_number","message","timestamp","function_name","log_level"]
+ process_array: false
+ max_depth: 1
+ target: ""
+ overwrite_keys: false
+ add_error_key: true
+</pre>
+</div>
+
+<p>
+The relevant documentation can be found here: <a href="https://www.elastic.co/guide/en/beats/filebeat/current/decode-json-fields.html">https://www.elastic.co/guide/en/beats/filebeat/current/decode-json-fields.html</a>
+</p>
+
+<p>
+After creating a new index in ElasticSearch and ingesting logs to this new index, the expected KQL behavior worked.
+</p>
+
+<p>
+The reason why I'm making this blog post is that it took me hours to find this documentation, as there seems to be about 1000 different ways to get this
+functionality, with a number of different caveats or options depending on your use case. I may just be showing my inexperience with ElasticSearch here,
+but decided to write something brief about this because it took me a while to track down.
+</p>
+
+<p>
+Note: This post isn't a knock against Elastic and their products. They solve a complex issue and give users a lot of options for how to manage, index, and
+search their data. I think given those options though, groking documentation can become time consuming and I wanted to try and offer a shortcut.
+</p>
+</div>
+</div>
+</div>
+</div>
+<div id="postamble" class="status">
+<p class="date">Created: 2022-01-14 Fri 10:55</p>
+<p class="validation"><a href="https://validator.w3.org/check?uri=referer">Validate</a></p>
+</div>
+</body>
+</html>
diff --git a/2022-01-13-es-json.org b/2022-01-13-es-json.org
new file mode 100644
index 0000000..b3171e7
--- /dev/null
+++ b/2022-01-13-es-json.org
@@ -0,0 +1,48 @@
+-*- org-mode -*-
+
+* How to parse single line JSON logs for ElasticSearch Cloud
+This is going to be a brief blog post, but wanted to jot down a few things as solving this "easy" issue has taken me the better part of 4 hours.
+** The Problem
+I have a number of weirdly formatted logs that developers would like to be able to easily search through and get insights from. The developers control this log format,
+but its an embedded environment and it's "non-trivial" to modify the format. I wrote a Perl script that will read in these developer logs and regex out
+key fields I'm interested in, transforming them like so (fake data):
+
+#+BEGIN_SRC shell
+# Original log line
+# LOG LEVEL # DATE & TIME # FUNCTION NAME/LINE NUMBER # LOG MESSAGE
+[DEBUG] 2020/9/10 - 13:59:23 | some_function_name 166: some log message
+
+# PARSED LOG LINE
+{"log_level":"Debug","timestamp":"2020-09-10T13:59:23","function_name":"some_function_name","line_number":"166","message":"some log message"}
+#+END_SRC
+
+After setting up this log parser and filebeat, I started processing these logs into a hosted ElasticSearch cloud instance. To my surprised, the JSON fields were
+not indexed, meaning I couldn't perform KQL searches like =timestamp:2020-09*= to get all log lines from that month.
+
+** The Solution
+To Elastic's credit, it's actually incredibly simple to get this behavior with filebeat, all I needed to do was add the following to the =/etc/filebeat/filebeat.yml=
+file under the =processors= field (This is on filebeat versions 7.x):
+
+#+BEGIN_SRC yaml
+processors:
+ - decode_json_fields:
+ fields: ["line_number","message","timestamp","function_name","log_level"]
+ process_array: false
+ max_depth: 1
+ target: ""
+ overwrite_keys: false
+ add_error_key: true
+#+END_SRC
+
+The relevant documentation can be found here: https://www.elastic.co/guide/en/beats/filebeat/current/decode-json-fields.html
+
+After creating a new index in ElasticSearch and ingesting logs to this new index, the expected KQL behavior worked.
+
+The reason why I'm making this blog post is that it took me hours to find this documentation, as there seems to be about 1000 different ways to get this
+functionality, with a number of different caveats or options depending on your use case. I may just be showing my inexperience with ElasticSearch here,
+but decided to write something brief about this because it took me a while to track down.
+
+Note: This post isn't a knock against Elastic and their products. They solve a complex issue and give users a lot of options for how to manage, index, and
+search their data. I think given those options though, groking documentation can become time consuming and I wanted to try and offer a shortcut.
+
+
diff --git a/2022-01-13-es-json.pretty.html b/2022-01-13-es-json.pretty.html
new file mode 100644
index 0000000..8accb3d
--- /dev/null
+++ b/2022-01-13-es-json.pretty.html
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+<!-- 2022-01-14 Fri 10:55 -->
+<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title></title>
+<meta name="generator" content="Org mode" />
+<style type="text/css">body{margin:40px
+auto;max-width:900px;line-height:1.6;font-size:18px;color:#444;padding:0
+10px}h1,h2,h3{line-height:1.2}</style>
+<script type="text/javascript">
+// @license magnet:?xt=urn:btih:e95b018ef3580986a04669f1b5879592219e2a7a&dn=public-domain.txt Public Domain
+<!--/*--><![CDATA[/*><!--*/
+ function CodeHighlightOn(elem, id)
+ {
+ var target = document.getElementById(id);
+ if(null != target) {
+ elem.classList.add("code-highlighted");
+ target.classList.add("code-highlighted");
+ }
+ }
+ function CodeHighlightOff(elem, id)
+ {
+ var target = document.getElementById(id);
+ if(null != target) {
+ elem.classList.remove("code-highlighted");
+ target.classList.remove("code-highlighted");
+ }
+ }
+ /*]]>*///-->
+// @license-end
+</script>
+</head>
+<body>
+<div id="content">
+<div id="table-of-contents">
+<h2>Table of Contents</h2>
+<div id="text-table-of-contents">
+<ul>
+<li><a href="#org4003f02">1. How to parse single line JSON logs for ElasticSearch Cloud</a>
+<ul>
+<li><a href="#org60dc23d">1.1. The Problem</a></li>
+<li><a href="#orgdfc5189">1.2. The Solution</a></li>
+</ul>
+</li>
+</ul>
+</div>
+</div>
+<p>
+</p>
+
+<div id="outline-container-org4003f02" class="outline-2">
+<h2 id="org4003f02"><span class="section-number-2">1</span> How to parse single line JSON logs for ElasticSearch Cloud</h2>
+<div class="outline-text-2" id="text-1">
+<p>
+This is going to be a brief blog post, but wanted to jot down a few things as solving this "easy" issue has taken me the better part of 4 hours.
+</p>
+</div>
+<div id="outline-container-org60dc23d" class="outline-3">
+<h3 id="org60dc23d"><span class="section-number-3">1.1</span> The Problem</h3>
+<div class="outline-text-3" id="text-1-1">
+<p>
+I have a number of weirdly formatted logs that developers would like to be able to easily search through and get insights from. The developers control this log format,
+but its an embedded environment and it's "non-trivial" to modify the format. I wrote a Perl script that will read in these developer logs and regex out
+key fields I'm interested in, transforming them like so (fake data):
+</p>
+
+<div class="org-src-container">
+<pre class="src src-shell"># Original log line
+# LOG LEVEL # DATE & TIME # FUNCTION NAME/LINE NUMBER # LOG MESSAGE
+[DEBUG] 2020/9/10 - 13:59:23 | some_function_name 166: some log message
+
+# PARSED LOG LINE
+{"log_level":"Debug","timestamp":"2020-09-10T13:59:23","function_name":"some_function_name","line_number":"166","message":"some log message"}
+</pre>
+</div>
+
+<p>
+After setting up this log parser and filebeat, I started processing these logs into a hosted ElasticSearch cloud instance. To my surprised, the JSON fields were
+not indexed, meaning I couldn't perform KQL searches like <code>timestamp:2020-09*</code> to get all log lines from that month.
+</p>
+</div>
+</div>
+
+<div id="outline-container-orgdfc5189" class="outline-3">
+<h3 id="orgdfc5189"><span class="section-number-3">1.2</span> The Solution</h3>
+<div class="outline-text-3" id="text-1-2">
+<p>
+To Elastic's credit, it's actually incredibly simple to get this behavior with filebeat, all I needed to do was add the following to the <code>/etc/filebeat/filebeat.yml</code>
+file under the <code>processors</code> field (This is on filebeat versions 7.x):
+</p>
+
+<div class="org-src-container">
+<pre class="src src-yaml">processors:
+ - decode_json_fields:
+ fields: ["line_number","message","timestamp","function_name","log_level"]
+ process_array: false
+ max_depth: 1
+ target: ""
+ overwrite_keys: false
+ add_error_key: true
+</pre>
+</div>
+
+<p>
+The relevant documentation can be found here: <a href="https://www.elastic.co/guide/en/beats/filebeat/current/decode-json-fields.html">https://www.elastic.co/guide/en/beats/filebeat/current/decode-json-fields.html</a>
+</p>
+
+<p>
+After creating a new index in ElasticSearch and ingesting logs to this new index, the expected KQL behavior worked.
+</p>
+
+<p>
+The reason why I'm making this blog post is that it took me hours to find this documentation, as there seems to be about 1000 different ways to get this
+functionality, with a number of different caveats or options depending on your use case. I may just be showing my inexperience with ElasticSearch here,
+but decided to write something brief about this because it took me a while to track down.
+</p>
+
+<p>
+Note: This post isn't a knock against Elastic and their products. They solve a complex issue and give users a lot of options for how to manage, index, and
+search their data. I think given those options though, groking documentation can become time consuming and I wanted to try and offer a shortcut.
+</p>
+</div>
+</div>
+</div>
+</div>
+<div id="postamble" class="status">
+<p class="author">Author: Simon Watson</p>
+<p class="date">Created: 2022-01-14 Fri 10:55</p>
+</div>
+</body>
+</html>