Merge pull request #50 from sivir/master

Delete supported params

Merge pull request #50 from sivir/master
8d89326f · Seth House · 9c0f18c4 · f03d88a7 · 8d89326f · 8d89326f
Commit 8d89326f authored 10 years ago by Seth House
--- a/README.rst
+++ b/README.rst
@@ -82,26 +82,3 @@ Remove the database called ``test``, normally created as part of a default
 MySQL installation.  This state is **not** included as part of the meta-state
 above as this name may conflict with a real database.
-Updating the supported parameters
-=================================
-The ``supported_params.yaml`` file contains the full listing of options that
-are acceptable in the MySQL options file.  On occassion, especially on new
-releases of MySQL, this file may need to be updated.  To update, run the
-supplied script (requires Python 3.x)::
-    ./scripts/parse_supported_params.py -o ./mysql/supported_params.yaml
-This script will scrape the options from the official MySQL documentation
-online, and thus requires web access.  Scraping is inherently brittle, though
-this script has been defensively coded, where possible.
-Once the ``supported_params.yaml`` file has been updated, commit the result to
-the repository.
-Support for new applications
----------------------------
-To add support for configuration of other MySQL applications, add the URL and
-section identifier into the relevant section of the script.  Consult the
-comments in the code to determine where your section should be added.
--- a/mysql/defaults.yaml
+++ b/mysql/defaults.yaml
 # vim: sts=2 ts=2 sw=2 et ai
+#
+# SET ALL PARAMS IN CONFIG SECTION USING UNDERSCORE, NOT HYPHEN
+# so that it works correctly
 {% load_yaml as rawmap %}
 Ubuntu:
  server: mysql-server
@@ -17,20 +20,20 @@ Ubuntu:
        nice: 0
      mysqld:
        user: mysql
-        pid-file: /var/run/mysqld/mysqld.pid
+        pid_file: /var/run/mysqld/mysqld.pid
        socket: /var/run/mysqld/mysqld.sock
        port: 3306
        basedir: /usr
        datadir: /var/lib/mysql
        tmpdir: /tmp
-        lc-messages-dir:  /usr/share/mysql
+        lc_messages_dir:  /usr/share/mysql
-        skip-external-locking: noarg_present
+        skip_external_locking: noarg_present
-        bind-address: 127.0.0.1
+        bind_address: 127.0.0.1
        key_buffer_size: 16M
        max_allowed_packet: 16M
        thread_stack: 192K
        thread_cache_size: 8
-        myisam-recover: BACKUP
+        myisam_recover: BACKUP
        query_cache_limit: 1M
        query_cache_size: 16M
        log_error: /var/log/mysql/error.log 
@@ -38,10 +41,10 @@ Ubuntu:
        max_binlog_size: 100M          
      mysqldump:
        quick: noarg_present
-        quote-names: noarg_present
+        quote_names: noarg_present
        max_allowed_packet: 16M
      isamchk:
-        key_buffer: 16M
+        key_buffer_size: 16M
    append: |
      !includedir /etc/mysql/conf.d/
 Debian:
@@ -61,30 +64,30 @@ Debian:
        nice: 0
      mysqld:
        user: mysql
-        pid-file: /var/run/mysqld/mysqld.pid
+        pid_file: /var/run/mysqld/mysqld.pid
        socket: /var/run/mysqld/mysqld.sock
        port: 3306
        basedir: /usr
        datadir: /var/lib/mysql
        tmpdir: /tmp
-        lc-messages-dir:  /usr/share/mysql
+        lc_messages_dir:  /usr/share/mysql
-        skip-external-locking: noarg_present
+        skip_external_locking: noarg_present
-        bind-address: 127.0.0.1
+        bind_address: 127.0.0.1
        key_buffer_size: 16M
        max_allowed_packet: 16M
        thread_stack: 192K
        thread_cache_size: 8
-        myisam-recover: BACKUP
+        myisam_recover: BACKUP
        query_cache_limit: 1M
        query_cache_size: 16M
        expire_logs_days: 10
        max_binlog_size: 100M          
      mysqldump:
        quick: noarg_present
-        quote-names: noarg_present
+        quote_names: noarg_present
        max_allowed_packet: 16M
      isamchk:
-        key_buffer: 16M
+        key_buffer_size: 16M
    append: |
      !includedir /etc/mysql/conf.d/
 CentOS:
@@ -96,15 +99,15 @@ CentOS:
    file: /etc/my.cnf
    sections:
      mysqld_safe:
-        log-error: /var/log/mysqld.log
+        log_error: /var/log/mysqld.log
-        pid-file: /var/run/mysqld/mysqld.pid
+        pid_file: /var/run/mysqld/mysqld.pid
      mysqld:
        datadir: /var/lib/mysql
        socket: /var/lib/mysql/mysql.sock
        user: mysql
        port: 3306
-        bind-address: 127.0.0.1
+        bind_address: 127.0.0.1
-        symbolic-links: 0
+        symbolic_links: 0
 RedHat:
  server: mysql-server
  client: mysql
@@ -114,15 +117,15 @@ RedHat:
    file: /etc/my.cnf
    sections:
      mysqld_safe:
-        log-error: /var/log/mysqld.log
+        log_error: /var/log/mysqld.log
-        pid-file: /var/run/mysqld/mysqld.pid
+        pid_file: /var/run/mysqld/mysqld.pid
      mysqld:
        datadir: /var/lib/mysql
        socket: /var/lib/mysql/mysql.sock
        user: mysql
        port: 3306
-        bind-address: 127.0.0.1
+        bind_address: 127.0.0.1
-        symbolic-links: 0
+        symbolic_links: 0
 Amazon:
  server: mysql-server
  client: mysql
@@ -132,15 +135,15 @@ Amazon:
    file: /etc/my.cnf
    sections:
      mysqld_safe:
-        log-error: /var/log/mysqld.log
+        log_error: /var/log/mysqld.log
-        pid-file: /var/run/mysqld/mysqld.pid
+        pid_file: /var/run/mysqld/mysqld.pid
      mysqld:
        datadir: /var/lib/mysql
        socket: /var/lib/mysql/mysql.sock
        user: mysql
        port: 3306
-        bind-address: 127.0.0.1
+        bind_address: 127.0.0.1
-        symbolic-links: 0
+        symbolic_links: 0
 Gentoo:
  server: dev-db/mysql
  client: dev-db/mysql
@@ -153,43 +156,43 @@ Gentoo:
        port: 3306
        socket: /var/run/mysqld/mysqld.sock
      mysql:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
-        default-character-set: utf8
+        default_character_set: utf8
      mysqladmin:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
-        default-character-set: utf8
+        default_character_set: utf8
      mysqlcheck:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
-        default-character-set: utf8
+        default_character_set: utf8
      mysqldump:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
-        default-character-set: utf8
+        default_character_set: utf8
      mysqlimport:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
-        default-character-set: utf8
+        default_character_set: utf8
      mysqlshow:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
-        default-character-set: utf8
+        default_character_set: utf8
      myisamchk:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
        key_buffer: 20M
        sort_buffer_size: 20M
        read_buffer: 2M
        write_buffer: 2M
      myisampack:
-        character-sets-dir: /usr/share/mysql/charsets
+        character_sets_dir: /usr/share/mysql/charsets
      mysqld_safe:
-        err-log: /var/log/mysql/mysql.err
+        err_log: /var/log/mysql/mysql.err
      mysqld:
-        character-set-server: utf8
+        character_set_server: utf8
        user: mysql
        port: 3306
        socket: /var/run/mysqld/mysqld.sock
-        pid-file: /var/run/mysqld/mysqld.pid
+        pid_file: /var/run/mysqld/mysqld.pid
-        log-error: /var/log/mysql/mysqld.err
+        log_error: /var/log/mysql/mysqld.err
        basedir: /usr
        datadir: /var/lib/mysql
-        skip-external-locking: noarg_present
+        skip_external_locking: noarg_present
        key_buffer_size: 16M
        max_allowed_packet: 1M
        table_open_cache: 64
@@ -199,9 +202,9 @@ Gentoo:
        read_rnd_buffer_size: 512K
        myisam_sort_buffer_size: 8M
        language: /usr/share/mysql/english
-        bind-address: 127.0.0.1
+        bind_address: 127.0.0.1
-        log-bin: noarg_present
+        log_bin: noarg_present
-        server-id: 1
+        server_id: 1
        tmpdir: /tmp/
        innodb_buffer_pool_size: 16M
        innodb_additional_mem_pool_size: 2M
@@ -221,7 +224,7 @@ Gentoo:
        read_buffer: 2M
        write_buffer: 2M
      mysqlhotcopy:
-        interactive-timeout: noarg_present
+        interactive_timeout: noarg_present
 FreeBSD:
  server: mysql56-server
  client: mysql56-client

--- a/mysql/files/my.cnf
+++ b/mysql/files/my.cnf
 # DO NOT CHANGE THIS FILE!
 # This config is generated by SALTSTACK
 # and all change will be overrided on next salt call
-{% from "mysql/defaults.yaml" import rawmap with context %}
+{#-
-{% from "mysql/supported_params.yaml" import supported_params with context %}
+===== FETCH DATA =====
-{%- set datamap = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %}
+-#}
-{%- for section_name, supparams in supported_params.items() %}
+{%- from "mysql/defaults.yaml" import rawmap with context -%}
-[{{ section_name }}]
+{%- from "mysql/supported_sections.yaml" import supported_sections with context -%}
-	{%- for allowedparam in supparams|default([]) %}
+{%- set datamap = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) -%}
-	{%- set indents = 40 - allowedparam|count %}
+{#-
-	{%- set mparam = salt['pillar.get']('mysql:server:'+section_name+':'+allowedparam, false) %}
+===== COMBINE DATA =====
-	{%- if mparam %}
+-#}
-		{%- if mparam == "noarg_present" %}
+{%- set goodParamList = datamap.config.sections -%}
-{{ allowedparam }}
+{%- for section_name in supported_sections -%}
-		{%- else %}
+	{%- set sectdict = datamap.config.sections[section_name] | default({}) -%}
-{{ allowedparam }}{{ '='|indent(indents, true) }} {{ mparam }} 
+	{%- for mparam, mvalue in salt['pillar.get']('mysql:server:'+section_name, {}).items() -%}
-		{%- endif %}
+		{%- set mparamUnderscore = mparam | replace('-','_') -%}
-	{%- else %}
+		{%- do sectdict.update({mparamUnderscore:mvalue}) -%}
-		{%- if datamap.config.sections[section_name] is defined %}
+	{%- endfor -%}
-		{%- if datamap.config.sections[section_name][allowedparam] is defined %}
+	{%- do goodParamList.update({section_name:sectdict}) -%}
-			{%- if datamap.config.sections[section_name][allowedparam] == "noarg_present" %}
+{%- endfor -%}
-{{ allowedparam }}
+{#-
-			{%- else %}
+===== PRINT DATA =====
-{{ allowedparam }}{{ '='|indent(indents, true) }} {{ datamap.config.sections[section_name][allowedparam] }} 
+-#}
-			{%- endif %}
+{%- for sname,sdata in goodParamList.items() -%}
-		{%- endif %}
+{%- if sdata %}
-		{%- endif %}
-	{%- endif %}
+[{{ sname }}]
+{%- for mparam, mvalue in sdata.items()|default([]) -%}
+{%- set indents = 40 - mparam|count %}
+{% if mvalue == "noarg_present" -%}
+{{ mparam }}
+{%- else -%}
+{{ mparam }}{{ '='|indent(indents, true) }} {{ mvalue }} 
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
 {%- endfor %}
-{% endfor %}
+{{ datamap.config.append | default('') }}
--- a/mysql/supported_sections.yaml
+++ b/mysql/supported_sections.yaml
+# vim
+{% load_yaml as supported_sections %}
+- client
+- mysql
+- mysqldump
+- mysqld_safe
+- mysqlhotcopy
+- mysqladmin
+- mysqlcheck
+- mysqlimport
+- mysqlshow
+- myisampack
+- myisamchk
+- isamchk
+- mysqld
+{% endload %}
--- a/pillar.example
+++ b/pillar.example
@@ -5,13 +5,12 @@ mysql:
    user: mysql
    # my.cnf sections changes 
    mysqld:
+      # you can use either underscore or hyphen in param names
      bind-address: 0.0.0.0
+      log_bin: /var/log/mysql/mysql-bin.log
      port: 3307
-      server-id: 1
+      binlog_do_db: foo
-      log-bin: mysql-bin
-      binlog-do-db: foo
      auto_increment_increment: 5
-      max_connect_errors: 4294967295
    mysql:
      # my.cnf param that not require value
      no-auto-rehash: noarg_present
@@ -28,7 +27,7 @@ mysql:
      load: False
  # Manage users
-  # you can get pillar for existent server using import_users.py script
+  # you can get pillar for existent server using scripts/import_users.py script
  user:
    frank:
      password: 'somepass'
@@ -58,5 +57,4 @@ mysql:
    server: mysql-server
    client: mysql-client
    service: mysql-service
-    config: /etc/mysql/my.cnf
    python: python-mysqldb
--- a/scripts/import_users.py
+++ b/scripts/import_users.py
+#!/usr/bin/env python
+"This script helps you to get mysql.user pillar from existent mysql server"
+import argparse
+import MySQLdb
+import re
+__author__ = "Egor Potiomkin"
+__version__ = "1.0"
+__email__ = "eg13reg@gmail.com"
+parser = argparse.ArgumentParser()
+parser.add_argument('host', metavar='IP', help='host where you want to get users')
+parser.add_argument('user', metavar='user', help='mysql user that can show grants')
+parser.add_argument('password', metavar='password', help='user password')
+args = parser.parse_args()
+# PARSE GRANTS 
+mysqlcon = MySQLdb.connect(host=args.host,user=args.user,passwd=args.password,db="mysql",use_unicode=True, charset='utf8')
+mysqlCur = mysqlcon.cursor(MySQLdb.cursors.DictCursor)
+mysqlCur.execute(r'''select user,host from mysql.user;''')
+rows = mysqlCur.fetchall()
+users = []
+for row in rows:
+	users.append({'name': row['user'], 'host': row['host']});
+mysqlCur = mysqlcon.cursor()
+grants = []
+for user in users:
+	q = r'''show grants for '%s'@'%s';''' % (user['name'], user['host'])
+	try:
+		user['grants'] = []
+		mysqlCur.execute(q)
+		rows = mysqlCur.fetchall()
+		for row in rows:
+			mpass = re.search(
+				r"""GRANT USAGE ON \*\.\* TO .* IDENTIFIED BY PASSWORD '(\*[A-F0-9]*)\'""",
+				row[0])
+			if mpass is None:
+				mgrant = re.search(
+					r"""GRANT ([\s,A-Z]+) ON `?([a-zA-Z0-9_\-*\\]*)`?\.`?([a-zA-Z0-9_\-*\\]*)`? TO .*""",
+					row[0])
+				if mgrant is not None:					
+					user['grants'].append({'grant': [x.strip() for x in mgrant.group(1).split(',')], 'database': mgrant.group(2).replace('\\',''), 'table': mgrant.group(3).replace('\\','')})
+				else:
+					print "ERROR: CAN NOT PARSE GRANTS: ",row[0]
+			else:
+				user['password'] = mpass.group(1)
+	except MySQLdb.DatabaseError:
+		print "Error while getting grants for '%s'@'%s'" % (user['name'], user['host'])
+#raise SystemExit
+# PRINT RESULT
+""" PRINT EXAMPLE
+mysql:
+  user:
+    username:
+      host: host
+      password_hash: '*2792A97371B2D17789364A22A9B35D180166571A'
+      databases:
+        - database: testbase
+          table: table1
+          grants: ['select']
+"""
+print "mysql:"
+print "  user:"
+for user in users:
+	print "    %s:" % user['name']
+	print "      host: '%s'" % user['host']
+	if ('password' in user):
+		print "      password_hash: '%s'" % user['password']
+	print "      databases:"
+	for grant in user['grants']:
+		print "        - database: '%s'" % grant['database']
+		print "          table: '%s'" % grant['table']
+		print "          grants: ['%s']" % "','".join(grant['grant']).lower()