From 84f9a5c41e806461c4193db0c7a0ef652b1b1357 Mon Sep 17 00:00:00 2001 From: Simon J Mudd Date: Thu, 20 Nov 2014 23:47:46 +0100 Subject: [PATCH] Various adjustments --- BUGS | 42 ++++++++++----- lib/common.go | 15 +++--- lib/log.go | 2 +- .../file_summary_by_instance_row.go | 51 +++++++++++-------- .../table_io_waits_summary_by_table_row.go | 50 ++++++++++-------- state/state.go | 18 +++---- 6 files changed, 107 insertions(+), 71 deletions(-) diff --git a/BUGS b/BUGS index 3a1eed7..48e62b4 100644 --- a/BUGS +++ b/BUGS @@ -1,7 +1,7 @@ Bugs ---- -See also NEW_FEATURES for things which probably need adding soon. +See also NEW_FEATURES for things which probably need adding soon. 1. Show the binlogs completely and match them properly by looking at the pattern defined in the server. Currently I'm using @@ -15,18 +15,34 @@ connection to speed things up? 4. Sometimes I see this: -pstop 0.0.12 - 8:46:41 xxxxxxxxxxxxx / 5.6.21-log, up 1d 22h 59m 16s [ABS] -Latency by Table Name (table_io_waits_summary_by_table) 239 rows -Table Name Latency %| Fetch Insert Update Delete -xxx.xxxxxxxx 5124.10 h +++.+%|100.0% -xxxxxx.xxxxxxxxxxxx 5124.10 h +++.+%|100.0% -xxxxxx.xxxxxxx 5124.10 h +++.+%|100.0% -xxxxxxxx.xxxxxxxxxx 5124.09 h +++.+%|100.0% -xxxxxxxx.xxxxxxxx 5124.09 h +++.+%|100.0% -xxxxxx.xxxx 00:07:49 49.2%| 98.6% 0.3% 1.1% -xxxxxxxxx.xxxxxxxxxxx 00:06:23 40.2%| 97.4% 2.4% 0.1% -xxxxxxxxx.xxxxxxxxxx 26.25 s 2.8%|114.2% 0.1% +++.+% 0.0% -xxxxxx.xxxxxxxx 15.71 s 1.6%| 99.6% 0.4% +pstop 0.0.12 - 8:46:41 xxxxxxxxxxxxx / 5.6.21-log, up 1d 22h 59m 16s [ABS] +Latency by Table Name (table_io_waits_summary_by_table) 239 rows +Table Name Latency %| Fetch Insert Update Delete +xxx.xxxxxxxx 5124.10 h +++.+%|100.0% +xxxxxx.xxxxxxxxxxxx 5124.10 h +++.+%|100.0% +xxxxxx.xxxxxxx 5124.10 h +++.+%|100.0% +xxxxxxxx.xxxxxxxxxx 5124.09 h +++.+%|100.0% +xxxxxxxx.xxxxxxxx 5124.09 h +++.+%|100.0% +xxxxxx.xxxx 00:07:49 49.2%| 98.6% 0.3% 1.1% +xxxxxxxxx.xxxxxxxxxxx 00:06:23 40.2%| 97.4% 2.4% 0.1% +xxxxxxxxx.xxxxxxxxxx 26.25 s 2.8%|114.2% 0.1% +++.+% 0.0% +xxxxxx.xxxxxxxx 15.71 s 1.6%| 99.6% 0.4% with a very long latency. Related to doing backups. Need to check the cause of the 5124.10h ++ issue. Probably a subtraction problem of some sort. + +5. Fix the filename encoding for example in sys: + +sys.x@0024host_summary_by_sta 882.29 us 0.2%| 3.5% 96.5%| 100 | 30 33.3% 66.7% +sys.x@0024host_summary_by_fil 861.65 us 0.2%| 3.6% 96.4%| 100 | 30 33.3% 66.7% +sys.x@0024user_summary_by_sta 719.61 us 0.1%| 3.2% 96.8%| 60 | 18 33.3% 66.7% +sys.x@0024innodb_buffer_stats 563.74 us 0.1%| 3.8% 96.2%| 60 | 18 33.3% 66.7% +sys.x@0024schema_index_statis 558.13 us 0.1%| 1.8% 98.2%| 20 | 6 33.3% 66.7% + +Should be + +sys.x$host_summary_by_statist 882.29 us 0.2%| 3.5% 96.5%| 100 | 30 33.3% 66.7% +sys.x$host_summary_by_file_io 861.65 us 0.2%| 3.6% 96.4%| 100 | 30 33.3% 66.7% +sys.x$user_summary_by_stages 719.61 us 0.1%| 3.2% 96.8%| 60 | 18 33.3% 66.7% +sys.x$innodb_buffer_stats 563.74 us 0.1%| 3.8% 96.2%| 60 | 18 33.3% 66.7% +sys.x$schema_index_statistics 558.13 us 0.1%| 1.8% 98.2%| 20 | 6 33.3% 66.7% diff --git a/lib/common.go b/lib/common.go index 1f21c7f..9d31075 100644 --- a/lib/common.go +++ b/lib/common.go @@ -11,6 +11,9 @@ import ( const ( myname = "pstop" copyright = "Copyright (C) 2014 Simon J Mudd " + i_1024_2 = 1024 * 1024 + i_1024_3 = 1024 * 1024 * 1024 + i_1024_4 = 1024 * 1024 * 1024 * 1024 ) // myround converts this floating value to the right width etc. @@ -103,15 +106,15 @@ func FormatAmount(amount uint64) string { return strconv.Itoa(int(amount)) } - if amount > (1024 * 1024 * 1024 * 1024) { + if amount > i_1024_4 { suffix = "P" - decimal_amount = float64(amount) / 1024 / 1024 / 1024 / 1024 - } else if amount > (1024 * 1024 * 1024) { + decimal_amount = float64(amount) / i_1024_4 + } else if amount > i_1024_3 { suffix = "G" - decimal_amount = float64(amount) / 1024 / 1024 / 1024 - } else if amount > (1024 * 1024) { + decimal_amount = float64(amount) / i_1024_3 + } else if amount > i_1024_2 { suffix = "M" - decimal_amount = float64(amount) / 1024 / 1024 + decimal_amount = float64(amount) / i_1024_2 } else if amount > 1024 { suffix = "k" decimal_amount = float64(amount) / 1024 diff --git a/lib/log.go b/lib/log.go index 0ae07d4..23c7c8e 100644 --- a/lib/log.go +++ b/lib/log.go @@ -35,7 +35,7 @@ func (logger *MyLogger) EnableLogging(enable_logging bool) bool { if err != nil { log.Fatal("Failed to open log file", logfile, ":", err) } - logger.logger = log.New(file, "", log.Ldate|log.Ltime|log.Lshortfile) + logger.logger = log.New(file, "", log.Ldate|log.Ltime) } return old_value } diff --git a/performance_schema/file_summary_by_instance/file_summary_by_instance_row.go b/performance_schema/file_summary_by_instance/file_summary_by_instance_row.go index 2488647..7c70f3b 100644 --- a/performance_schema/file_summary_by_instance/file_summary_by_instance_row.go +++ b/performance_schema/file_summary_by_instance/file_summary_by_instance_row.go @@ -8,6 +8,7 @@ import ( "log" "regexp" "sort" + // "strconv" "time" "github.com/sjmudd/pstop/lib" @@ -165,23 +166,20 @@ func (t file_summary_by_instance_rows) totals() file_summary_by_instance_row { // clean up the given path reducing redundant stuff and return the clean path func cleanup_path(path string) string { - // foo/../bar --> bar perl: $new =~ s{[^/]+/\.\./}{/}; - // foo/./bar --> foo/bar perl: $new =~ s{/\./}{}; + // foo/../bar --> foo/bar perl: $new =~ s{[^/]+/\.\./}{/}; + // /./ --> / perl: $new =~ s{/\./}{}; // // --> / perl: $new =~ s{//}{/}; const ( - double_slash_re = `//` - slash_dot_slash_re = `/\./` + one_or_the_other_re = `/(\.)?/` slash_dot_dot_slash_re = `[^/]+/\.\./` ) + r1 := regexp.MustCompile(one_or_the_other_re) + r2 := regexp.MustCompile(slash_dot_dot_slash_re) + for { orig_path := path - - r := regexp.MustCompile(double_slash_re) - path = r.ReplaceAllString(path, "") - r = regexp.MustCompile(slash_dot_slash_re) - path = r.ReplaceAllString(path, "") - r = regexp.MustCompile(slash_dot_dot_slash_re) - path = r.ReplaceAllString(path, "") + path = r1.ReplaceAllString(path, "/") + path = r2.ReplaceAllString(path, "/") if orig_path == path { // no change so give up break } @@ -194,25 +192,34 @@ func cleanup_path(path string) string { // This simpler name may also merge several different filenames into one. func (t file_summary_by_instance_row) simple_name(global_variables map[string]string) string { const ( - auto_cnf_re = `/auto\.cnf$` - binlog_re = `/binlog\.(\d{6}|index)$` - charset_re = `/share/charsets/Index\.xml$` - db_opt_re = `/db\.opt$` - error_msg_re = `/share/[^/]+/errmsg\.sys$` - ibdata_re = `/ibdata\d+$` - redo_log_re = `/ib_logfile\d+$` - pid_file_re = `/[^/]+\.pid$` - // relay_log_re = `/mysql-relay-bin.(\d{6}|index)$` - relative_path_re = `^\.\./` + auto_cnf_re = `/auto\.cnf$` + binlog_re = `/binlog\.(\d{6}|index)$` + charset_re = `/share/charsets/Index\.xml$` current_dir_re = `^\./` + db_opt_re = `/db\.opt$` + encoded_re = `@(\d{4})` // FIXME - add me to catch @0024 --> $ for example + error_msg_re = `/share/[^/]+/errmsg\.sys$` + ibdata_re = `/ibdata\d+$` + part_table_re = `(.+)#P#p\d+` + pid_file_re = `/[^/]+\.pid$` + redo_log_re = `/ib_logfile\d+$` + relative_path_re = `^\.\./` slowlog_re = `/slowlog$` table_file_re = `/([^/]+)/([^/]+)\.(frm|ibd|MYD|MYI|CSM|CSV|par)$` temp_table_re = `#sql-[0-9_]+` - part_table_re = `(.+)#P#p\d+` ) path := t.FILE_NAME + // FIXME and make this work. + // re4 := regexp.MustCompile(encoded_re) + // if m4 := re4.FindStringSubmatch(path); m4 != nil { + // if value, err := strconv.ParseInt(m4[1], 16, 16); err != nil { + // // missing replace @.... with char(value) in path + // + // } + // } + // this should probably be ordered from most expected regexp to least re := regexp.MustCompile(table_file_re) if m1 := re.FindStringSubmatch(path); m1 != nil { diff --git a/performance_schema/table_io_waits_summary_by_table/table_io_waits_summary_by_table_row.go b/performance_schema/table_io_waits_summary_by_table/table_io_waits_summary_by_table_row.go index cde898f..2551e0d 100644 --- a/performance_schema/table_io_waits_summary_by_table/table_io_waits_summary_by_table_row.go +++ b/performance_schema/table_io_waits_summary_by_table/table_io_waits_summary_by_table_row.go @@ -126,22 +126,31 @@ func (this *table_io_waits_summary_by_table_row) add(other table_io_waits_summar this.COUNT_WRITE += other.COUNT_WRITE } +// subtract the countable values in one row from another func (this *table_io_waits_summary_by_table_row) subtract(other table_io_waits_summary_by_table_row) { - this.SUM_TIMER_WAIT -= other.SUM_TIMER_WAIT - this.SUM_TIMER_FETCH -= other.SUM_TIMER_FETCH - this.SUM_TIMER_INSERT -= other.SUM_TIMER_INSERT - this.SUM_TIMER_UPDATE -= other.SUM_TIMER_UPDATE - this.SUM_TIMER_DELETE -= other.SUM_TIMER_DELETE - this.SUM_TIMER_READ -= other.SUM_TIMER_READ - this.SUM_TIMER_WRITE -= other.SUM_TIMER_WRITE - - this.COUNT_STAR -= other.COUNT_STAR - this.COUNT_FETCH -= other.COUNT_FETCH - this.COUNT_INSERT -= other.COUNT_INSERT - this.COUNT_UPDATE -= other.COUNT_UPDATE - this.COUNT_DELETE -= other.COUNT_DELETE - this.COUNT_READ -= other.COUNT_READ - this.COUNT_WRITE -= other.COUNT_WRITE + // check for issues here (we have a bug) and log it + // - this situation should not happen so there's a logic bug somewhere else + if this.SUM_TIMER_WAIT >= other.SUM_TIMER_WAIT { + this.SUM_TIMER_WAIT -= other.SUM_TIMER_WAIT + this.SUM_TIMER_FETCH -= other.SUM_TIMER_FETCH + this.SUM_TIMER_INSERT -= other.SUM_TIMER_INSERT + this.SUM_TIMER_UPDATE -= other.SUM_TIMER_UPDATE + this.SUM_TIMER_DELETE -= other.SUM_TIMER_DELETE + this.SUM_TIMER_READ -= other.SUM_TIMER_READ + this.SUM_TIMER_WRITE -= other.SUM_TIMER_WRITE + + this.COUNT_STAR -= other.COUNT_STAR + this.COUNT_FETCH -= other.COUNT_FETCH + this.COUNT_INSERT -= other.COUNT_INSERT + this.COUNT_UPDATE -= other.COUNT_UPDATE + this.COUNT_DELETE -= other.COUNT_DELETE + this.COUNT_READ -= other.COUNT_READ + this.COUNT_WRITE -= other.COUNT_WRITE + } else { + lib.Logger.Println("WARNING: table_io_waits_summary_by_table_row.subtract() - subtraction problem! (not subtracting)") + lib.Logger.Println("this=", this) + lib.Logger.Println("other=", other) + } } func (t table_io_waits_summary_by_table_rows) totals() table_io_waits_summary_by_table_row { @@ -233,17 +242,18 @@ func (t table_io_waits_summary_by_table_rows) Sort(want_latency bool) { // remove the initial values from those rows where there's a match // - if we find a row we can't match ignore it func (this *table_io_waits_summary_by_table_rows) subtract(initial table_io_waits_summary_by_table_rows) { - i_by_name := make(map[string]int) + initial_by_name := make(map[string]int) // iterate over rows by name for i := range initial { - i_by_name[initial[i].name()] = i + initial_by_name[initial[i].name()] = i } for i := range *this { - if _, ok := i_by_name[(*this)[i].name()]; ok { - initial_i := i_by_name[(*this)[i].name()] - (*this)[i].subtract(initial[initial_i]) + this_name := (*this)[i].name() + if _, ok := initial_by_name[this_name]; ok { + initial_index := initial_by_name[this_name] + (*this)[i].subtract(initial[initial_index]) } } } diff --git a/state/state.go b/state/state.go index 1b4b54f..90c3f7d 100644 --- a/state/state.go +++ b/state/state.go @@ -177,7 +177,7 @@ func (state *State) DisplayNext() { func (state State) displayHeading() { state.displayLine0() - state.displayLine1() + state.displayDescription() } func (state State) displayLine0() { @@ -208,17 +208,19 @@ func (state State) displayLine0() { state.screen.PrintAt(0, 0, top_line) } -func (state State) displayLine1() { +func (state State) displayDescription() { + description := "UNKNOWN" + switch state.show { case showLatency, showOps: - state.screen.PrintAt(0, 1, state.tiwsbt.Description()) + description = state.tiwsbt.Description() case showIO: - state.screen.PrintAt(0, 1, state.fsbi.Description()) + description = state.fsbi.Description() case showLocks: - state.screen.PrintAt(0, 1, state.tlwsbt.Description()) - default: - state.screen.PrintAt(0, 1, "UNKNOWN") + description = state.tlwsbt.Description() } + + state.screen.PrintAt(0, 1, description) } func (state *State) displayOpsOrLatency() { @@ -305,8 +307,6 @@ func (state *State) SetWantRelativeStats(want_relative_stats bool) { state.fsbi.SetWantRelativeStats(want_relative_stats) state.tlwsbt.SetWantRelativeStats(state.want_relative_stats) state.tiwsbt.SetWantRelativeStats(state.want_relative_stats) - - state.Display() } // if there's a better way of doing this do it better ... -- 2.20.1