crashing: /proc/613/exe: Permission denied

Hi,
should I worry about the following?:

How-To-Repeat:

021018 13:19:33 mysqld started
crashing: /proc/613/exe: Permission denied
BFD load failed..
/usr/local/mysql/libexec/mysqld: ready for connections

# ls -la /proc/613
total 0
dr-xr-xr-x 3 mysql mysql 0 Oct 18 13:21 .
dr-xr-xr-x 61 root root 0 Jan 3 1997 ..
-r--r--r-- 1 root root 0 Oct 18 13:21 cmdline
-r--r--r-- 1 root root 0 Oct 18 13:21 cpu
lrwxrwxrwx 1 root root 0 Oct 18 13:21 cwd -> /data/mysql
-r-------- 1 root root 0 Oct 18 13:21 environ
lrwxrwxrwx 1 root root 0 Oct 18 13:21 exe -> /usr/local/mysql-BK-20021018/libexec/mysqld
dr-x------ 2 root root 0 Oct 18 13:21 fd
-r--r--r-- 1 root root 0 Oct 18 13:21 maps
-rw------- 1 root root 0 Oct 18 13:21 mem
-r--r--r-- 1 root root 0 Oct 18 13:21 mounts
lrwxrwxrwx 1 root root 0 Oct 18 13:21 root -> /
-r--r--r-- 1 root root 0 Oct 18 13:21 stat
-r--r--r-- 1 root root 0 Oct 18 13:21 statm
-r--r--r-- 1 root root 0 Oct 18 13:21 status
# ls -la /usr/local/mysql-BK-20021018/libexec/mysqld
-rwxr-xr-x 1 bioadmin mysql 3072359 Oct 18 13:00 /usr/local/mysql-BK-20021018/libexec/mysqld
#


Another trial using strace gives me:

[pid 456] getpid() = 456
[pid 456] write(7, "456", 3) = 3
[pid 456] close(7) = 0
[pid 456] getpid() = 456
[pid 456] getpid() = 456
[pid 456] rt_sigaction(SIGSEGV, {0x81ad5d8, [], SA_ONESHOT|SA_SIGINFO|0x4000000}, NULL, 8) = 0
[pid 456] rt_sigaction(SIGUSR1, {0x81ad5d8, [], SA_SIGINFO|0x4000000}, NULL, 8) = 0
[pid 456] brk(0) = 0x8302000
[pid 456] stat64("/proc/456/exe", 0xbf7ff604) = -1 EACCES (Permission denied)
[pid 456] brk(0) = 0x8302000
[pid 456] brk(0x8303000) = 0x8303000
[pid 456] brk(0) = 0x8303000
[pid 456] brk(0x8307000) = 0x8307000
[pid 456] open("/proc/456/exe", O_RDONLY) = -1 EACCES (Permission denied)
[pid 456] write(2, "crashing: /proc/456/exe: Permiss"..., 43) = 43
[pid 456] write(2, "BFD load failed..\n", 18) = 18
[pid 456] kill(453, SIGRTMIN) = 0


BTW: Inspecting the trace log tells me, there are a lost of PATH's
hardcoded into binaries, and very often it tries to open ./anything before
using $BINDIR/anything or using similar variable set by mysqld_safe. Is
that expected?


I'm starting /etc/rc3.d/mysql.server as root from current
working directory /usr/local. It seems, I'd better do
"cd /usr/local/mysql":

stat64("/usr/local/data/mysql", 0xbffff7ac) = -1 ENOENT (No such file or directory)
stat64("./share/mysql/english/errmsg.sys", 0xbffff78c) = -1 ENOENT (No such file or directory)
stat64("./bin/mysqld", 0xbffff6dc) = -1 ENOENT (No such file or directory)
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
stat64("./var/mysql/db.frm", 0xbffff6ec) = -1 ENOENT (No such file or directory)
stat64("./share/mysql/english/errmsg.sys", 0xbffff6cc) = -1 ENOENT (No such file or directory)
stat64("./libexec/mysqld", 0xbffff61c) = -1 ENOENT (No such file or directory)
[...]
stat64("/usr/local/mysql/libexec/mysqld-max", 0xbffff76c) = -1 ENOENT (No such file or directory)
[...]
stat64("./bin/my_print_defaults", 0xbffff76c) = -1 ENOENT (No such file or directory)
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
stat64("/usr/local/mysql/bin/my_print_defaults", {st_dev=makedev(8, 4), st_ino=292883, st_mode=S_IFREG|0755, st_nlink=1, st_uid=1
001, st_gid=1002, st_blksize=4096, st_blocks=1416, st_size=718017, st_atime=2002/10/18-13:10:02, st_mtime=2002/10/18-12:59:55, st
_ctime=2002/10/18-13:02:08}) = 0
[...]
[pid 416] open("/etc/my.cnf", O_RDONLY|O_LARGEFILE) = 3
[...]
[pid 453] open("./mysql/func.frm", O_RDONLY|O_LARGEFILE) = 7
[pid 453] read(7, "\376\1\7\t\3\0\0\20\1\0\0000\0\0\347\0\303\0\0\0\0\0\0"..., 64) = 64
[pid 453] _llseek(7, 64, [64], SEEK_SET) = 0
[pid 453] read(7, "//\0\0 \0\0", 7) = 7
[pid 453] _llseek(7, 4096, [4096], SEEK_SET) = 0
[pid 453] read(7, "\1\1\23\0\n\0\2@\0\1\1\200\2\0\0\0\0@\0\377PRIMARY\377"..., 30) = 30
[pid 453] pread(7, "\377 "..., 195, 4327) = 195
[pid 453] _llseek(7, 8192, [8192], SEEK_SET) = 0
[pid 453] read(7, "\301\1\0\20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"... , 288) = 288
[pid 453] read(7, "M\0\5\4\2\24) "..., 161) = 161
[pid 453] close(7) = 0
[pid 453] getcwd("/data/mysql", 4096) = 12
[pid 453] lstat64("/data/mysql/mysql", {st_dev=makedev(8, 33), st_ino=6635521, st_mode=S_IFDIR|0700, st_nlink=2, st_uid=1002, s
t_gid=1002, st_blksize=4096, st_blocks=8, st_size=4096, st_atime=2002/10/18-06:28:32, st_mtime=2002/07/16-16:20:07, st_ctime=2002
/07/16-16:20:07}) = 0
[pid 453] lstat64("/data/mysql/mysql/func.MYI", {st_dev=makedev(8, 33), st_ino=6635538, st_mode=S_IFREG|0660, st_nlink=1, st_ui
d=1002, st_gid=1002, st_blksize=4096, st_blocks=8, st_size=1024, st_atime=2002/10/18-13:10:06, st_mtime=2002/07/16-16:20:07, st_c
time=2002/07/16-16:20:07}) = 0
[pid 453] open("/data/mysql/mysql/func.MYI", O_RDWR|O_LARGEFILE) = 7
[pid 453] read(7, "\376\376\7\1\0\2\1i\0\260\0d\0\304\0\1\0\0\1\0\10\1\0\0"... , 24) = 24
[pid 453] readlink("./mysql/func.MYI", 0xbfffeaec, 511) = -1 EINVAL (Invalid argument)
[pid 453] readlink("./mysql/func.MYD", 0xbfffe8ec, 511) = -1 EINVAL (Invalid argument)
[pid 453] _llseek(7, 0, [0], SEEK_SET) = 0
[pid 453] read(7, "\376\376\7\1\0\2\1i\0\260\0d\0\304\0\1\0\0\1\0\10\1\0\0"... , 361) = 361
[pid 453] open("./mysql/func.MYD", O_RDWR|O_LARGEFILE) = 8
[pid 453] getpid() = 453
[pid 453] _llseek(8, 0, [0], SEEK_END) = 0
[pid 453] close(7) = 0
[pid 453] close(8) = 0
[pid 453] getcwd("/data/mysql", 4096) = 12
[pid 453] lstat64("/data/mysql/master.info", 0xbfffe30c) = -1 ENOENT (No such file or directory)
[pid 453] access("./master.info", F_OK) = -1 ENOENT (No such file or directory)
[pid 453] fstat64(1, {st_dev=makedev(8, 33), st_ino=65543, st_mode=S_IFREG|0644, st_nlink=1, st_uid=1002, st_gid=0, st_blksize=
4096, st_blocks=296, st_size=127298, st_atime=2002/10/18-13:17:08, st_mtime=2002/10/18-13:17:09, st_ctime=2002/10/18-13:17:09}) =
0
[pid 453] old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40000000
[pid 453] write(1, "/usr/local/mysql/libexec/mysqld:"..., 55) = 55
[...]

But those are probably only of interrest until my.cnf is found.




linux 2.4.20-pre5-ac1, mysql-4.0-BK-$today
--
Martin Mokrejs ,
PGP5.0i key is at http://www.natur.cuni.cz/~mmokrejs
MIPS / Institute for Bioinformatics
GSF - National Research Center for Environment and Health
Ingolstaedter Landstrasse 1, D-85764 Neuherberg, Germany
tel.: +49-89-3187 3683 , fax: +49-89-3187 3585


------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread12786@lists.mysql.com
To unsubscribe, e-mail

crashing: /proc/613/exe: Permission denied

Hi!

>>>>> "Martin" == Martin MOKREJ writes:

Martin> Hi,
Martin> should I worry about the following?:

Martin> How-To-Repeat:

Martin> 021018 13:19:33 mysqld started
Martin> crashing: /proc/613/exe: Permission denied
Martin> BFD load failed..
Martin> /usr/local/mysql/libexec/mysqld: ready for connections



Martin> # ls -la /usr/local/mysql-BK-20021018/libexec/mysqld
Martin> -rwxr-xr-x 1 bioadmin mysql 3072359 Oct 18 13:00 /usr/local/mysql-BK-20021018/libexec/mysqld

Do you really mean that you get first an error message but mysqld
starts as normally ?

This is not normal and could be a bug in the thread library you are
using. The 'crashing:' text doesn't come from MySQL but from system
library.

If you could try to compile MySQL with debugging and try to figure out
what system call causes this we could try to figure out a workaround
for this.

Martin> Another trial using strace gives me:


Unfortunately this doesn't tell us which library/system call that
prints the error message.

Martin> BTW: Inspecting the trace log tells me, there are a lost of PATH's
Martin> hardcoded into binaries, and very often it tries to open ./anything before
Martin> using $BINDIR/anything or using similar variable set by mysqld_safe. Is
Martin> that expected?

Can you give us some examples of this ?
Especially when searching after my.cnf files mysqld will try some
pre-determinate paths, but appart from that this shouldn't happen.

Martin> I'm starting /etc/rc3.d/mysql.server as root from current
Martin> working directory /usr/local. It seems, I'd better do
Martin> "cd /usr/local/mysql":

The mysqld_safe script assumes that you are in the directory where
mysqld is installed; This enables you to have many different versions
of MySQL on the same machine (which makes it easy to upgrade and
test new versions)

Martin> stat64("/usr/local/data/mysql", 0xbffff7ac) = -1 ENOENT (No such file or directory)
Martin> stat64("./share/mysql/english/errmsg.sys", 0xbffff78c) = -1 ENOENT (No such file or directory)

The above is normal when you start mysqld_safe from the wrong place.



Martin> But those are probably only of interrest until my.cnf is found.

Actually you should not get any errors like the above if mysqld fines
a my.cnf file as my.cnf files are searched for first. After this
mysqld should use the paths from the my.cnf file instead of any
internal paths.

Regards,
Monty

--
For technical support contracts, goto https://order.mysql.com/
__ ___ ___ ____ __
/ |/ /_ __/ __/ __ \/ / Mr. Michael Widenius
/ /|_/ / // /\ \/ /_/ / /__ MySQL AB, CTO
/_/ /_/\_, /___/\___\_\___/ Helsinki, Finland
<___/ www.mysql.com

------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread12807@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

On Sun, 20 Oct 2002, Michael Widenius wrote:

MW> Martin> How-To-Repeat:
MW>
MW> Martin> 021018 13:19:33 mysqld started
MW> Martin> crashing: /proc/613/exe: Permission denied
MW> Martin> BFD load failed..
MW> Martin> /usr/local/mysql/libexec/mysqld: ready for connections
MW>
MW>
MW>
MW> Martin> # ls -la /usr/local/mysql-BK-20021018/libexec/mysqld
MW> Martin> -rwxr-xr-x 1 bioadmin mysql 3072359 Oct 18 13:00 /usr/local/mysql-BK-20021018/libexec/mysqld
MW>
MW> Do you really mean that you get first an error message but mysqld
MW> starts as normally ?

Yes and runs 3 days already.

MW> This is not normal and could be a bug in the thread library you are
MW> using. The 'crashing:' text doesn't come from MySQL but from system
MW> library.
MW>
MW> If you could try to compile MySQL with debugging and try to figure out
MW> what system call causes this we could try to figure out a workaround
MW> for this.

I'll try, after to make bitkeeper to clone the mysq-4.0 source tree. Thge
old I had to delete because of space. Hmm, don't you want to check into
the tree configure script? ;)


I see now in the $datadir 2 files:

REPORT: CURRENT PROCESS:459, THREAD:2051
--------- STACK TRACE FOR PROCESS 459, THREAD 2051 ---------------

FAULT ADDR: 00000000
EIP: 0023:[<081c31c9>] ESP: 002b:bf5ff938
EAX: fffffffc EBX: bf5ff984 ECX: 00000008 EDX: 0000001f
ESI: bf5ffc00 EDI: bf5ff984 EBP: bf5ff954 DS: 002b ES: 002b
STACK TRACE:
0x081c31c9: ???? (bf5ff984, 0, bf5ff984, 81ac1e6, 0, 0) + 90
0x081ad269: ???? (bf5ffc00, 0, bf5ffa44, 81aa345, 8300fc0, 8300fcc) + 20
0x081aa3fc: ???? (82c0bf4, 82c05fc, bf5ffa84, 8080234, 8300fc0, 8300fc0) + 10
0x08080312: ???? (8300fc0, 1, bf5ffb34, 808966b, 8300fcc, 0) + a0
0x08089680: ???? (8300fc0, 0, bf5ffd28, 81ab113, 81ab100, 0) + a0
0x081ab1d8: ???? (bf5ffc00, bf5ffc00, 0, 0, 0, 0) + 40a003ec



--------- SEGV in PROCESS 453, THREAD 1024 ---------------

FAULT ADDR: 00000008
EIP: 0023:[<081d885f>] ESP: 002b:bffff560
EAX: 08300fb8 EBX: 00000000 ECX: 08300fb8 EDX: 0000005a
ESI: 08300fb8 EDI: 000008b8 EBP: bffff588 DS: 002b ES: 002b
STACK TRACE:
0x081d885f: ???? (8300fc0, bffff738, bffff69c, 0, 0, 0)
0x081d85e3: ???? (8300fc0, 0, bffff5d8, 81abe4c, 8300fc0, bffff738)
0x08193766: ???? (8300fc0, 5590000, bffff618, 8080298, 82c05fc, 2b)
0x0807b13b: ???? (8300fc0, 3, 0, 0, 2, 0) + 10
0x080802b8: ???? (8300fc0, 0, 0, 0, 0, 0)
0x08080273: ???? (2, 0, 0, 2b, 2b, 0) + 7c
0x081ad5a8: ???? (2, 0, 0, 2b, 2b, 0) + 448
0x081c3148: ???? (0, bffffcef, 829a860, 821d640, bffffc20, bffffb78) + 40
0x080816fe: ???? (23, 82ef2b8, bffffc20, 82168e0, 0, bffffbd8) + 1c
0x081c01d5: ???? (8080da4, 6, bffffc04, 80480b4, 82168e0, 0) + 40000408


I've never resolved linux kernel/thread crashes, any quick help?

--
Martin Mokrejs ,
PGP5.0i key is at http://www.natur.cuni.cz/~mmokrejs
MIPS / Institute for Bioinformatics
GSF - National Research Center for Environment and Health
Ingolstaedter Landstrasse 1, D-85764 Neuherberg, Germany
tel.: +49-89-3187 3683 , fax: +49-89-3187 3585



------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread12810@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

On Sun, 20 Oct 2002, Michael Widenius wrote:

Hi Michael,

MW> Martin> How-To-Repeat:
MW>
MW> Martin> 021018 13:19:33 mysqld started
MW> Martin> crashing: /proc/613/exe: Permission denied
MW> Martin> BFD load failed..
MW> Martin> /usr/local/mysql/libexec/mysqld: ready for connections
MW>
MW>
MW>
MW> Martin> # ls -la /usr/local/mysql-BK-20021018/libexec/mysqld
MW> Martin> -rwxr-xr-x 1 bioadmin mysql 3072359 Oct 18 13:00 /usr/local/mysql-BK-20021018/libexec/mysqld
MW>
MW> Do you really mean that you get first an error message but mysqld
MW> starts as normally ?

Yes, it works fine, but I don't see a debug file created.

Linux 2.4.20-pre10-ac1 #1 SMP Tue Oct 15 21:06:15 CEST 2002 i686 unknown unknown GNU/Linux



/usr/local/mysql-BK20021104# ./libexec/mysqld --datadir=/data/mysql2 --basedir=/usr/local/mysql-BK20021104 --pid-file=/usr/local/mysql/var/kulan.pidd --port=3307 --socket=/tmp/mysql2.sock --debug=/var/tmp/mysqldubug.log
crashing: /proc/21211/exe: Permission denied
BFD load failed..
../libexec/mysqld: ready for connections


021104 14:27:26 ./libexec/mysqld: Normal shutdown

021104 14:27:27 ./libexec/mysqld: Shutdown Complete


User time 0.10, System time 1.11
Maximum resident set size 0, Integral resident set size 0
Non-physical pagefaults 49443, Physical pagefaults 316, Swaps 0
Blocks in 0 out 0, Messages in 0 out 0, Signals 0
Voluntary context switches 0, Involuntary context switches 0
Maximum memory usage: 202076524 bytes (197341k)
/usr/local/mysql-BK20021104#


/scratch/mysql-4.0# gdb ./sql/mysqld
GNU gdb 5.2.1
Copyright 2002 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i686-pc-linux-gnu"...
(gdb) set args --datadir=/data/mysql2 --basedir=/usr/local/mysql-BK20021104 --pid-file=/usr/local/mysql/var/kulan.pidd --port=3307 --socket=/tmp/mysql2.sock --debug=/var/tmp/mysqldubug.log
(gdb) b mysqld.cc:2039
Breakpoint 1 at 0x8084190: file mysqld.cc, line 2039.
(gdb) run
Starting program: /usr/scratch/mysql-4.0/sql/mysqld --datadir=/data/mysql2 --basedir=/usr/local/mysql-BK20021104 --pid-file=/usr/local/mysql/var/kulan.pidd --port=3307 --socket=/tmp/mysql2.sock --debug=/var/tmp/mysqldubug.log

Breakpoint 1, main (argc=36, argv=0x8329da0) at mysqld.cc:2039
2039 start_signal_handler(); // Creates pidfile
(gdb) s
start_signal_handler () at mysqld.cc:1488
1488 (void) pthread_attr_init(&thr_attr);
(gdb) s
1490 pthread_attr_setscope(&thr_attr,PTHREAD_SCOPE_SYSTEM);
(gdb)
1491 (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHE D);
(gdb)
1492 if (!(opt_specialflag & SPECIAL_NO_PRIOR))
(gdb)
1493 my_pthread_attr_setprio(&thr_attr,INTERRUPT_PRIOR);
(gdb)
my_pthread_attr_setprio (attr=0xbffffb08, priority=10) at my_pthread.c:68
68 bzero((char*) &tmp_sched_param,sizeof(tmp_sched_param));
Current language: auto; currently c
(gdb)
69 tmp_sched_param.sched_priority=priority;
(gdb)
70 VOID(pthread_attr_setschedparam(attr,&tmp_sched_param));
(gdb)
start_signal_handler () at mysqld.cc:1494
1494 pthread_attr_setstacksize(&thr_attr,32768);
Current language: auto; currently c++
(gdb)
1497 (void) pthread_mutex_lock(&LOCK_thread_count);
(gdb)
safe_mutex_lock (mp=0x82fa6c0, file=0x824975c "mysqld.cc", line=1497) at thr_mutex.c:53
53 pthread_mutex_lock(&mp->global);
Current language: auto; currently c
(gdb)
54 if (mp->count > 0 && pthread_equal(pthread_self(),mp->thread))
(gdb)
61 pthread_mutex_unlock(&mp->global);
(gdb)
62 error=pthread_mutex_lock(&mp->mutex);
(gdb)
63 if (error || (error=pthread_mutex_lock(&mp->global)))
(gdb)
70 if (mp->count++)
(gdb)
77 mp->thread=pthread_self();
(gdb)
78 mp->file= (char*) file;
(gdb)
79 mp->line=line;
(gdb)
80 pthread_mutex_unlock(&mp->global);
(gdb)
81 return error;
(gdb)
start_signal_handler () at mysqld.cc:1498
1498 if ((error=pthread_create(&signal_thread,&thr_attr,signal_hand, 0)))
Current language: auto; currently c++
(gdb)
1504 (void) pthread_cond_wait(&COND_thread_count,&LOCK_thread_count);
(gdb) crashing: /proc/21712/exe: Permission denied
BFD load failed..

safe_cond_wait (cond=0x82fa9cc, mp=0x82fa6c0, file=0x824975c "mysqld.cc", line=1504) at thr_mutex.c:125
125 pthread_mutex_lock(&mp->global);
Current language: auto; currently c
(gdb)
126 if (mp->count == 0)
(gdb)
132 if (!pthread_equal(pthread_self(),mp->thread))
(gdb)
140 if (mp->count-- != 1)
(gdb)
147 pthread_mutex_unlock(&mp->global);
(gdb)
148 error=pthread_cond_wait(cond,&mp->mutex);
(gdb)
149 pthread_mutex_lock(&mp->global);
(gdb)
150 if (error)
(gdb)
156 if (mp->count++)
(gdb)
164 mp->thread=pthread_self();
(gdb)
165 mp->file= (char*) file;
(gdb)
164 mp->thread=pthread_self();
(gdb)
165 mp->file= (char*) file;
(gdb)
166 mp->line=line;
(gdb)
167 pthread_mutex_unlock(&mp->global);
(gdb)
168 return error;
(gdb)
start_signal_handler () at mysqld.cc:1505
1505 pthread_mutex_unlock(&LOCK_thread_count);
Current language: auto; currently c++
(gdb)
safe_mutex_unlock (mp=0x82fa6c0, file=0x824975c "mysqld.cc", line=1505) at thr_mutex.c:88
88 pthread_mutex_lock(&mp->global);
Current language: auto; currently c
(gdb)
89 if (mp->count == 0)
(gdb)
96 if (!pthread_equal(pthread_self(),mp->thread))
(gdb)
103 mp->count--;
(gdb)
108 error=pthread_mutex_unlock(&mp->mutex);
(gdb)
109 if (error)
(gdb)
116 pthread_mutex_unlock(&mp->global);
(gdb)
117 return error;
(gdb)
start_signal_handler () at mysqld.cc:1507
1507 (void) pthread_attr_destroy(&thr_attr);
Current language: auto; currently c++
(gdb)
1508 DBUG_VOID_RETURN;
(gdb)
_db_return_ (_line_=1508, _sfunc_=0xbffffaf8, _sfile_=0xbffffafc, _slevel_=0xbffffb00) at dbug.c:795
795 if (!_no_db_)
Current language: auto; currently c
(gdb)
797 int save_errno=errno;
(gdb)
798 if (!(state=code_state()))
(gdb)
code_state () at dbug.c:352
352 struct st_my_thread_var *tmp=my_thread_var;
(gdb)
_my_thread_var () at my_thr_init.c:192
192 struct st_my_thread_var *tmp=
(gdb)

/usr/scratch/mysql-4.0/sql/mysqld: ready for connections


[...]




MW>
MW> This is not normal and could be a bug in the thread library you are
MW> using. The 'crashing:' text doesn't come from MySQL but from system
MW> library.
MW>
MW> If you could try to compile MySQL with debugging and try to figure out
MW> what system call causes this we could try to figure out a workaround
MW> for this.
MW>
MW> Martin> Another trial using strace gives me:
MW>
MW>
MW> Unfortunately this doesn't tell us which library/system call that
MW> prints the error message.

[pid 21263] old_mmap(0xbf7f7000, 36864, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xbf7f7000
[pid 21263] mprotect(0xbf7f7000, 4096, PROT_NONE) = 0
[pid 21263] clone(child_stack=0xbf7ffbf8, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|0x21) = 21264
[pid 21263] kill(21262, SIGRTMIN) = 0
[pid 21262] --- SIGRTMIN (Real-time signal 0) ---
[pid 21263] poll(
[pid 21262] <... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call)
[pid 21262] sigreturn() = ? (mask now [HUP QUIT PIPE TERM TSTP RTMIN])
[pid 21262] rt_sigprocmask(SIG_SETMASK, NULL, [HUP QUIT PIPE TERM TSTP RTMIN], 8) = 0
[pid 21262] rt_sigsuspend([HUP QUIT PIPE TERM TSTP]
[pid 21264] getpid() = 21264
[pid 21264] rt_sigprocmask(SIG_SETMASK, [HUP QUIT PIPE TERM TSTP RTMIN], NULL, 8) = 0
[pid 21264] sched_setscheduler(0x5310, 0, 0xbf7ffd28) = 0
[pid 21264] rt_sigprocmask(SIG_BLOCK, [ALRM], NULL, 8) = 0
[pid 21264] rt_sigaction(SIGALRM, {0x81d7c68, [], 0x4000000}, {0x81d7c68, [], 0x4000000}, 8) = 0
[pid 21264] rt_sigprocmask(SIG_UNBLOCK, [ALRM], NULL, 8) = 0
[pid 21264] rt_sigprocmask(SIG_UNBLOCK, [ALRM], NULL, 8) = 0
[pid 21264] open("/usr/local/mysql/var/kulan.pidd", O_WRONLY|O_CREAT|O_LARGEFILE, 0664) = 7
[pid 21264] getpid() = 21264
[pid 21264] write(7, "21264", 5) = 5
[pid 21264] close(7) = 0
[pid 21264] getpid() = 21264
[pid 21264] getpid() = 21264
[pid 21264] rt_sigaction(SIGSEGV, {0x81d7d48, [], SA_ONESHOT|SA_SIGINFO|0x4000000}, NULL, 8) = 0
[pid 21264] rt_sigaction(SIGUSR1, {0x81d7d48, [], SA_SIGINFO|0x4000000}, NULL, 8) = 0
[pid 21264] brk(0) = 0x833c000
[pid 21264] stat64("/proc/21264/exe", 0xbf7ff5f4) = -1 EACCES (Permission denied)
[pid 21264] brk(0) = 0x833c000
[pid 21264] brk(0x833d000) = 0x833d000
[pid 21264] brk(0) = 0x833d000
[pid 21264] brk(0x833e000) = 0x833e000
[pid 21264] brk(0) = 0x833e000
[pid 21264] brk(0x8342000) = 0x8342000
[pid 21264] open("/proc/21264/exe", O_RDONLY) = -1 EACCES (Permission denied)
[pid 21264] write(2, "crashing: /proc/21264/exe: Permi"..., 45crashing: /proc/21264/exe: Permission denied
) = 45
[pid 21264] write(2, "BFD load failed..\n", 18BFD load failed..
) = 18
[pid 21264] kill(21262, SIGRTMIN) = 0
[pid 21264] rt_sigprocmask(SIG_BLOCK, [HUP QUIT ALRM TERM TSTP], NULL, 8) = 0
[pid 21264] rt_sigaction(SIGQUIT, {0x81d7c68, ~[], 0x4000000},
[pid 21262] --- SIGRTMIN (Real-time signal 0) ---
[pid 21264] <... rt_sigaction resumed> NULL, 8) = 0
[pid 21264] rt_sigaction(SIGTSTP, {0x81d7c68, ~[], 0x4000000}, NULL, 8) = 0


MW>
MW> Martin> BTW: Inspecting the trace log tells me, there are a lost of PATH's
MW> Martin> hardcoded into binaries, and very often it tries to open ./anything before
MW> Martin> using $BINDIR/anything or using similar variable set by mysqld_safe. Is
MW> Martin> that expected?
MW>
MW> Can you give us some examples of this ?
MW> Especially when searching after my.cnf files mysqld will try some
MW> pre-determinate paths, but appart from that this shouldn't happen.

OK, something a bit different what I meant previously, but let's think of this:

[pid 21262] open("./mysql/func.frm", O_RDONLY|O_LARGEFILE) = 7
[pid 21262] read(7, "\376\1\7\t\3\0\0\20\1\0\0000\0\0\347\0\303\0\0\0\0\0\0"..., 64) = 64
[pid 21262] _llseek(7, 64, [64], SEEK_SET) = 0
[pid 21262] read(7, "//\0\0 \0\0", 7) = 7
[pid 21262] _llseek(7, 4096, [4096], SEEK_SET) = 0
[pid 21262] read(7, "\1\1\23\0\n\0\2@\0\1\1\200\2\0\0\0\0@\0\377PRIMARY\377"..., 30) = 30
[pid 21262] pread(7, "\377 "..., 195, 4327) = 195
[pid 21262] _llseek(7, 8192, [8192], SEEK_SET) = 0
[pid 21262] read(7, "\301\1\0\20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"... , 288) = 288
[pid 21262] read(7, "M\0\5\4\2\24) "..., 161) = 161
[pid 21262] close(7) = 0
[pid 21262] getcwd("/data/mysql2", 4096) = 13

--------------------^ we do getcwd() foreach mysql.* table, although it seems we already are
in $datadir.

[pid 21262] lstat64("/data/mysql2/mysql", {st_dev=makedev(8, 33), st_ino=57901057, st_mode=S_IFDIR|0700, st_nlink=2, st_uid=1002, st_gid=1002, st_blksize=4096, st_blocks=8, st_size=4096, st_atime=2002/11/04-06:26:45, st_mtime=2002/07/16-16:20:07, st_ctime=2002/11/01-06:10:41}) = 0
[pid 21262] lstat64("/data/mysql2/mysql/func.MYI", {st_dev=makedev(8, 33), st_ino=57901065, st_mode=S_IFREG|0660, st_nlink=1, st_uid=1002, st_gid=1002, st_blksize=4096, st_blocks=8, st_size=1024, st_atime=2002/11/04-14:25:53, st_mtime=2002/07/16-16:20:07, st_ctime=2002/11/01-06:10:41}) = 0
[pid 21262] open("/data/mysql2/mysql/func.MYI", O_RDWR|O_LARGEFILE) = 7
[pid 21262] read(7, "\376\376\7\1\0\2\1i\0\260\0d\0\304\0\1\0\0\1\0\10\1\0\0"... , 24) = 24
[pid 21262] readlink("./mysql/func.MYI", 0xbfffea4c, 511) = -1 EINVAL (Invalid argument)
[pid 21262] readlink("./mysql/func.MYD", 0xbfffe84c, 511) = -1 EINVAL (Invalid argument)

------------------------------------------------------------ ---^ ?

[pid 21262] _llseek(7, 0, [0], SEEK_SET) = 0
[pid 21262] read(7, "\376\376\7\1\0\2\1i\0\260\0d\0\304\0\1\0\0\1\0\10\1\0\0"... , 361) = 361
[pid 21262] open("./mysql/func.MYD", O_RDWR|O_LARGEFILE) = 8


MW>
MW> Martin> I'm starting /etc/rc3.d/mysql.server as root from current
MW> Martin> working directory /usr/local. It seems, I'd better do
MW> Martin> "cd /usr/local/mysql":
MW>
MW> The mysqld_safe script assumes that you are in the directory where
MW> mysqld is installed; This enables you to have many different versions
MW> of MySQL on the same machine (which makes it easy to upgrade and
MW> test new versions)
MW>
MW> Martin> stat64("/usr/local/data/mysql", 0xbffff7ac) = -1 ENOENT (No such file or directory)
MW> Martin> stat64("./share/mysql/english/errmsg.sys", 0xbffff78c) = -1 ENOENT (No such file or directory)
MW>
MW> The above is normal when you start mysqld_safe from the wrong place.

OK, I'll keep it in mind.

MW>
MW>
MW>
MW> Martin> But those are probably only of interrest until my.cnf is found.
MW>
MW> Actually you should not get any errors like the above if mysqld fines
MW> a my.cnf file as my.cnf files are searched for first. After this
MW> mysqld should use the paths from the my.cnf file instead of any
MW> internal paths.


open("/etc/my.cnf", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_dev=makedev(8, 1), st_ino=104, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=16, st_size=5592, st_atime=2002/11/04-14:30:06, st_mtime=2002/08/30-14:56:28, st_ctime=2002/08/30-14:56:28}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40000000
read(3, "# Example mysql config file for "..., 4096) = 4096
read(3, "ing Innobase tables\n#innodb_data"..., 4096) = 1496
read(3, "", 4096) = 0
close(3) = 0
munmap(0x40000000, 4096) = 0
open("/usr/local/mysql/var/my.cnf", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/root/.my.cnf", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)

Shouldn't we be more paranoid at least about the file permission on these configfiles?
We read just anything, even world writable and owned by nobody. ;)


brk(0x832b000) = 0x832b000
readlink("/usr/local/mysql-BK20021104", 0x82f9f00, 511) = -1 EINVAL (Invalid argument)


[...]



stat64("/usr/local/mysql-BK20021104/share/mysql/charsets/cp1 257.conf", {st_dev=makedev(8, 4), st_ino=81331, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=50, st_blksize=4096, st_blocks=16, st_size=4399, st_atime=2002/11/04-14:21:59, st_mtime=2002/11/04-14:21:59, st_ctime=2002/11/04-14:21:59}) = 0
stat64("/usr/local/mysql-BK20021104/share/mysql/charsets/lat in5.conf", {st_dev=makedev(8, 4), st_ino=81346, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=50, st_blksize=4096, st_blocks=16, st_size=4545, st_atime=2002/11/04-14:21:59, st_mtime=2002/11/04-14:21:59, st_ctime=2002/11/04-14:21:59}) = 0
sched_setscheduler(0x530e, 0, 0xbffffaf8) = -1 EINVAL (Invalid argument)

----------------------------------------------------^ ?

sched_get_priority_max(0) = 0
sched_get_priority_min(0) = 0
getrlimit(0x7, 0xbffffab4) = 0
setrlimit(RLIMIT_NOFILE, {rlim_cur=32878, rlim_max=32878}) = 0
getrlimit(0x7, 0xbffffab4) = 0
getpid() = 21262
time(NULL) = 1036416662
open("/usr/local/mysql-BK20021104/share/mysql/english/errmsg .sys", O_RDONLY|O_LARGEFILE) = 3


--
Martin Mokrejs ,
PGP5.0i key is at http://www.natur.cuni.cz/~mmokrejs
MIPS / Institute for Bioinformatics
GSF - National Research Center for Environment and Health
Ingolstaedter Landstrasse 1, D-85764 Neuherberg, Germany
tel.: +49-89-3187 3683 , fax: +49-89-3187 3585



------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread12896@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

Hi!

Sorry for the delay, I have been very busy lately...
I will now answer all your open questions in this thread.

>>>>> "Martin" == Martin MOKREJ writes:



Martin> I see now in the $datadir 2 files:

Martin> REPORT: CURRENT PROCESS:459, THREAD:2051
Martin> --------- STACK TRACE FOR PROCESS 459, THREAD 2051 ---------------

Martin> FAULT ADDR: 00000000
Martin> EIP: 0023:[<081c31c9>] ESP: 002b:bf5ff938
Martin> EAX: fffffffc EBX: bf5ff984 ECX: 00000008 EDX: 0000001f
Martin> ESI: bf5ffc00 EDI: bf5ff984 EBP: bf5ff954 DS: 002b ES: 002b
Martin> STACK TRACE:
Martin> 0x081c31c9: ???? (bf5ff984, 0, bf5ff984, 81ac1e6, 0, 0) + 90
Martin> 0x081ad269: ???? (bf5ffc00, 0, bf5ffa44, 81aa345, 8300fc0, 8300fcc) + 20
Martin> 0x081aa3fc: ???? (82c0bf4, 82c05fc, bf5ffa84, 8080234, 8300fc0, 8300fc0) + 10
Martin> 0x08080312: ???? (8300fc0, 1, bf5ffb34, 808966b, 8300fcc, 0) + a0
Martin> 0x08089680: ???? (8300fc0, 0, bf5ffd28, 81ab113, 81ab100, 0) + a0
Martin> 0x081ab1d8: ???? (bf5ffc00, bf5ffc00, 0, 0, 0, 0) + 40a003ec



Martin> I've never resolved linux kernel/thread crashes, any quick help?

I have never got a stack trace like the above, so I can't help you
much with this.

If the above comes from mysql, you can get some information about this
by pasting the stack trace result to 'resolve_stack_trace' that you
can find in the extra directory:

nm -n mysqld > mysqld.sym
resolve_stack_trace mysqld.sym
0x081c31c9

etc...

Regards,
Monty

------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread12956@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

Hi!

>>>>> "Martin" == Martin MOKREJ writes:

Martin> On Sun, 20 Oct 2002, Michael Widenius wrote:
Martin> Hi Michael,

MW> Martin> How-To-Repeat:
MW>
MW> Martin> 021018 13:19:33 mysqld started
MW> Martin> crashing: /proc/613/exe: Permission denied
MW> Martin> BFD load failed..



Martin> /scratch/mysql-4.0# gdb ./sql/mysqld

Martin> start_signal_handler () at mysqld.cc:1498
Martin> 1498 if ((error=pthread_create(&signal_thread,&thr_attr,signal_hand, 0)))
Martin> Current language: auto; currently c++
Martin> (gdb)
Martin> 1504 (void) pthread_cond_wait(&COND_thread_count,&LOCK_thread_count);
Martin> (gdb) crashing: /proc/21712/exe: Permission denied
Martin> BFD load failed..

The above means that either something is wrong with your
pthread_cond_wait() call or that because of some strange reason
the signal handler died.

You could try to verify this by setting a break point at 'signal_hand'
and single step through this code.



MW> Martin> BTW: Inspecting the trace log tells me, there are a lost of PATH's
MW> Martin> hardcoded into binaries, and very often it tries to open ./anything before
MW> Martin> using $BINDIR/anything or using similar variable set by mysqld_safe. Is
MW> Martin> that expected?
MW>
MW> Can you give us some examples of this ?
MW> Especially when searching after my.cnf files mysqld will try some
MW> pre-determinate paths, but appart from that this shouldn't happen.

Martin> OK, something a bit different what I meant previously, but let's think of this:

Martin> [pid 21262] open("./mysql/func.frm", O_RDONLY|O_LARGEFILE) = 7
Martin> [pid 21262] read(7, "\376\1\7\t\3\0\0\20\1\0\0000\0\0\347\0\303\0\0\0\0\0\0"..., 64) = 64
Martin> [pid 21262] _llseek(7, 64, [64], SEEK_SET) = 0
Martin> [pid 21262] read(7, "//\0\0 \0\0", 7) = 7
Martin> [pid 21262] _llseek(7, 4096, [4096], SEEK_SET) = 0
Martin> [pid 21262] read(7, "\1\1\23\0\n\0\2@\0\1\1\200\2\0\0\0\0@\0\377PRIMARY\377"..., 30) = 30
Martin> [pid 21262] pread(7, "\377 "..., 195, 4327) = 195
Martin> [pid 21262] _llseek(7, 8192, [8192], SEEK_SET) = 0
Martin> [pid 21262] read(7, "\301\1\0\20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"... , 288) = 288
Martin> [pid 21262] read(7, "M\0\5\4\2\24) "..., 161) = 161
Martin> [pid 21262] close(7) = 0
Martin> [pid 21262] getcwd("/data/mysql2", 4096) = 13

Martin> --------------------^ we do getcwd() foreach mysql.* table, although it seems we already are
Martin> in $datadir.

MySQL has to call 'getwcd()' to be able to resolve the name to an
absolute path (needed for the internal locking handling).

MySQL will cache the result from getwcd() for future calls, so this
isn't that bad.

Martin> [pid 21262] readlink("./mysql/func.MYD", 0xbfffe84c, 511) = -1 EINVAL (Invalid argument)


Martin> ------------------------------------------------------------ ---^ ?

The above is ok, as func.MYD is not a symbolic link.



MW> Martin> But those are probably only of interrest until my.cnf is found.
MW>
MW> Actually you should not get any errors like the above if mysqld fines
MW> a my.cnf file as my.cnf files are searched for first. After this
MW> mysqld should use the paths from the my.cnf file instead of any
MW> internal paths.


Martin> open("/etc/my.cnf", O_RDONLY|O_LARGEFILE) = 3
Martin> fstat64(3, {st_dev=makedev(8, 1), st_ino=104, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=16, st_size=5592, st_atime=2002/11/04-14:30:06, st_mtime=2002/08/30-14:56:28, st_ctime=2002/08/30-14:56:28}) = 0
Martin> old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40000000
Martin> read(3, "# Example mysql config file for "..., 4096) = 4096
Martin> read(3, "ing Innobase tables\n#innodb_data"..., 4096) = 1496
Martin> read(3, "", 4096) = 0
Martin> close(3) = 0
Martin> munmap(0x40000000, 4096) = 0
Martin> open("/usr/local/mysql/var/my.cnf", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
Martin> open("/root/.my.cnf", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)

Martin> Shouldn't we be more paranoid at least about the file permission on these configfiles?
Martin> We read just anything, even world writable and owned by nobody. ;)

For the clients this is ok, as one can anyway change all values in the
~/my.cnf file.

I don't see any reason to slow down things by adding extra checks for
the defaults file; Most programs that read option files does the
same.

Martin> stat64("/usr/local/mysql-BK20021104/share/mysql/charsets/lat in5.conf", {st_dev=makedev(8, 4), st_ino=81346, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=50, st_blksize=4096, st_blocks=16, st_size=4545, st_atime=2002/11/04-14:21:59, st_mtime=2002/11/04-14:21:59, st_ctime=2002/11/04-14:21:59}) = 0
Martin> sched_setscheduler(0x530e, 0, 0xbffffaf8) = -1 EINVAL (Invalid argument)

Martin> ----------------------------------------------------^ ?

The above is ok and just means that you didn't configure MySQL with
support for the above language.

As MySQL does however allow you to add this support later, by just
copying the file in place, MySQL has to check this on startup to know
if the language is supported or not.

Regards,
Monty

--
For technical support contracts, goto https://order.mysql.com/
__ ___ ___ ____ __
/ |/ /_ __/ __/ __ \/ / Mr. Michael Widenius
/ /|_/ / // /\ \/ /_/ / /__ MySQL AB, CTO
/_/ /_/\_, /___/\___\_\___/ Helsinki, Finland
<___/ www.mysql.com

------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread12957@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

On Mon, 11 Nov 2002, Michael Widenius wrote:

>
> Hi!
>
> Sorry for the delay, I have been very busy lately...
> I will now answer all your open questions in this thread.
>
> >>>>> "Martin" == Martin MOKREJ writes:
>
>
>
> Martin> I see now in the $datadir 2 files:
>
> Martin> REPORT: CURRENT PROCESS:459, THREAD:2051
> Martin> --------- STACK TRACE FOR PROCESS 459, THREAD 2051 ---------------
>
> Martin> FAULT ADDR: 00000000
> Martin> EIP: 0023:[<081c31c9>] ESP: 002b:bf5ff938
> Martin> EAX: fffffffc EBX: bf5ff984 ECX: 00000008 EDX: 0000001f
> Martin> ESI: bf5ffc00 EDI: bf5ff984 EBP: bf5ff954 DS: 002b ES: 002b
> Martin> STACK TRACE:
> Martin> 0x081c31c9: ???? (bf5ff984, 0, bf5ff984, 81ac1e6, 0, 0) + 90
> Martin> 0x081ad269: ???? (bf5ffc00, 0, bf5ffa44, 81aa345, 8300fc0, 8300fcc) + 20
> Martin> 0x081aa3fc: ???? (82c0bf4, 82c05fc, bf5ffa84, 8080234, 8300fc0, 8300fc0) + 10
> Martin> 0x08080312: ???? (8300fc0, 1, bf5ffb34, 808966b, 8300fcc, 0) + a0
> Martin> 0x08089680: ???? (8300fc0, 0, bf5ffd28, 81ab113, 81ab100, 0) + a0
> Martin> 0x081ab1d8: ???? (bf5ffc00, bf5ffc00, 0, 0, 0, 0) + 40a003ec
>
>
>
> Martin> I've never resolved linux kernel/thread crashes, any quick help?
>
> I have never got a stack trace like the above, so I can't help you
> much with this.

Actually, to resolve system stack traces, this seems to be the right way:

From: Randy.Dunlap
Date: 2001-09-20 15:02:45
Subject: Re: __alloc_pages: 0-order allocation failed still in -pre12

Andrea Arcangeli wrote:
>
> Can you also resolve "c012e052" so we know who's allocating those pages
> just in case?

It's trivial to do that, of course, but if someone needs an
automated way to do it (several times, easy lookup), you can
try http://www.osdlab.org/sw_resources/scripts/ksysmap .

Usage is: ksysmap [system_map_file] offset

and it spits out address/symbol before offset, exact match if
present, and address/symbol after offset.

Example:

[rddunlap@dragon linux]$ ksysmap ./System.map-249acpi c012e052
ksysmap: searching './System.map-249acpi' for 'c012e052'

c012df20 T sys_truncate
c012e052 ..... <<<<<
c012e0a0 T sys_ftruncate


>
> If the above comes from mysql, you can get some information about this
> by pasting the stack trace result to 'resolve_stack_trace' that you
> can find in the extra directory:
>
> nm -n mysqld > mysqld.sym
> resolve_stack_trace mysqld.sym
> 0x081c31c9
>
> etc...

Yes, it works. I've resolved stack trace created today:

root@kulan:~# cat /data/mysql/mysqld-7935-87107-15643.backtrace
--------- SEGV in PROCESS 15643, THREAD 87107 ---------------

FAULT ADDR: fffffffc
EIP: 0023:[<081d8f97>] ESP: 002b:b75ff738
EAX: fffffffc EBX: b75ff934 ECX: 00000008 EDX: b75ffc00
ESI: fffffffc EDI: b75ffc00 EBP: b75ff760 DS: 002b ES: 002b
STACK TRACE:
0x081d8f97: ???? (b75ff934, b75ffc00, b75ff934, 0, 0, 0) + 10
0x081d4ab5: ???? (fffffffc, 82fa6d8, bd5263e8, cc50, b3, 1f80) + 130
0xb75ff7a5: ???? (b75ff934, 82fa6d8, 82fa6d8, b75ff9b4, 81d79c1, b75ff934) + 4
0x081ed939: ???? (b75ff934, 0, b75ff934, 81d91d7, 0, 0) + 90
0x081d79d9: ???? (b75ffc00, ffffff8f, b75ff9f4, 81d4ab5, 82fa6c0, b75ffa68) + 20
0x081d4b6c: ???? (82faf58, 82fa6d8, b75ffa34, 81c8c7d, 82fa6c0, 15443) + 20
0x081c8c8d: ???? (82faf58, 82fa6c0, 824975c, 4a6, 832b090, 93736c4) + 20
0x08082c99: ???? (93736b8, 1, b75ffb34, 808ce40, 93736c4, 0) + a0
0x0808ce55: ???? (96cb310, 0, b75ffd28, 81d5883, 81d5870, 0) + a0
0x081d5948: ???? (b75ffc00, b75ffc00, 0, 0, 0, 0) + 48a003ec
root@kulan:~# ls -la /data/mysql/mysqld-7935-87107-15643.backtrace
-rw------- 1 mysql mysql 949 Nov 20 10:48 /data/mysql/mysqld-7935-87107-15643.backtrace
root@kulan:~# nm -n /usr/local/mysql/libexec/mysqld > mysqld.sym
root@kulan:~# /usr/local/mysql/bin/resolve_stack_dump -s mysqld.sym -n /data/mysql/mysqld-7935-87107-15643.backtrace
--------- SEGV in PROCESS 15643, THREAD 87107 ---------------

FAULT ADDR: fffffffc
EIP: 0023:[<081d8f97>] ESP: 002b:b75ff738
EAX: fffffffc EBX: b75ff934 ECX: 00000008 EDX: b75ffc00
ESI: fffffffc EDI: b75ffc00 EBP: b75ff760 DS: 002b ES: 002b
STACK TRACE:
0x81d8f97 __pthread_lock + 39
0x81d4ab5 pthread_cond_wait + 173
0xb75ff7a5 _end + -1355978691
0x81ed939 sigsuspend + 25
0x81d79d9 __pthread_wait_for_restart_signal + 69
0x81d4b6c pthread_cond_wait + 356
0x81c8c8d safe_cond_wait + 241
0x8082c99 end_thread__FP3THDb + 265
0x808ce55 handle_one_connection__FPv + 973
0x81d5948 pthread_start_thread + 216
root@kulan:~#

The system was running linux-2.4.20-pre10-ac2 on i686.
mysql-BK20021104-debug

--
Martin Mokrejs ,
PGP5.0i key is at http://www.natur.cuni.cz/~mmokrejs
MIPS / Institute for Bioinformatics
GSF - National Research Center for Environment and Health
Ingolstaedter Landstrasse 1, D-85764 Neuherberg, Germany
tel.: +49-89-3187 3683 , fax: +49-89-3187 3585



------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread13050@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

On Mon, 11 Nov 2002, Michael Widenius wrote:

Hi,

> >>>>> "Martin" == Martin MOKREJ writes:
>
> Martin> On Sun, 20 Oct 2002, Michael Widenius wrote:
> Martin> Hi Michael,
>
> MW> Martin> How-To-Repeat:
> MW>
> MW> Martin> 021018 13:19:33 mysqld started
> MW> Martin> crashing: /proc/613/exe: Permission denied
> MW> Martin> BFD load failed..
>
>
>
> Martin> /scratch/mysql-4.0# gdb ./sql/mysqld
>
> Martin> start_signal_handler () at mysqld.cc:1498
> Martin> 1498 if ((error=pthread_create(&signal_thread,&thr_attr,signal_hand, 0)))
> Martin> Current language: auto; currently c++
> Martin> (gdb)
> Martin> 1504 (void) pthread_cond_wait(&COND_thread_count,&LOCK_thread_count);
> Martin> (gdb) crashing: /proc/21712/exe: Permission denied
> Martin> BFD load failed..
>
> The above means that either something is wrong with your
> pthread_cond_wait() call or that because of some strange reason
> the signal handler died.
>
> You could try to verify this by setting a break point at 'signal_hand'
> and single step through this code.

(gdb) set args --basedir=/usr/local/mysql --datadir=/data/mysql2 --user=mysql --pid-file=/data/mysql/kulan.pid2 --skip-locking open-files-limit=30000 --port=3307
(gdb) b signal_hand
Breakpoint 1 at 0x8080cfd: file mysqld.cc, line 1544.
(gdb) r
Starting program: /usr/scratch/mysql-4.0/sql/mysqld --basedir=/usr/local/mysql --datadir=/data/mysql2 --user=mysql --pid-file=/data/mysql/kulan.pid2 --skip-locking open-files-limit=30000 --port=3307

Program received signal SIGTRAP, Trace/breakpoint trap.
0x081c1869 in sigsuspend ()
(gdb) s
Single stepping until exit from function sigsuspend,
which has no line number information.


Program received signal SIGINT, Interrupt.
0x081c1869 in sigsuspend ()
(gdb) where
#0 0x081c1869 in sigsuspend ()
#1 0x081ab8f9 in __pthread_wait_for_restart_signal ()
#2 0x081a8c1c in pthread_cond_wait ()
#3 0x08080cca in start_signal_handler () at mysqld.cc:1525
#4 0x08081837 in main (argc=37, argv=0x82ed478) at mysqld.cc:2075
#5 0x081be865 in __libc_start_main ()
(gdb) s
Single stepping until exit from function sigsuspend,
which has no line number information.


The mysqld does not respond to queries, strace return no output (like
no activity?). Sending kill -CONT doesn't help, even when gdb is detached.

I gave it one more trial, and set breakpoint into sigsuspend:

(gdb) set args --basedir=/usr/local/mysql --datadir=/data/mysql2 --user=mysql --pid-file=/data/mysql/kulan.pid2 --skip-locking open-files-limit=30000 --port=3307
(gdb) r
Starting program: /usr/scratch/mysql-4.0/sql/mysqld --basedir=/usr/local/mysql --datadir=/data/mysql2 --user=mysql --pid-file=/data/mysql/kulan.pid2 --skip-locking open-files-limit=30000 --port=3307

Breakpoint 1, 0x081c1857 in sigsuspend ()
(gdb) crashing: /proc/4257/exe: Permission denied
BFD load failed..

(gdb) where
#0 0x081c1857 in sigsuspend ()
#1 0x081ab8f9 in __pthread_wait_for_restart_signal ()
#2 0x081ab283 in pthread_create ()
#3 0x08080c8f in start_signal_handler () at mysqld.cc:1519
#4 0x08081837 in main (argc=37, argv=0x82ed478) at mysqld.cc:2075
#5 0x081be865 in __libc_start_main ()
(gdb) s
Single stepping until exit from function sigsuspend,
which has no line number information.
0x081ab8f9 in __pthread_wait_for_restart_signal ()
(gdb)
Single stepping until exit from function __pthread_wait_for_restart_signal,
which has no line number information.
0x081ab283 in pthread_create ()
(gdb)
Single stepping until exit from function pthread_create,
which has no line number information.
start_signal_handler () at mysqld.cc:1525
1525 (void) pthread_cond_wait(&COND_thread_count,&LOCK_thread_count);
(gdb)

Breakpoint 1, 0x081c1857 in sigsuspend ()
(gdb)
Single stepping until exit from function sigsuspend,
which has no line number information.

Breakpoint 1, 0x081c1857 in sigsuspend ()
(gdb)
Single stepping until exit from function sigsuspend,
which has no line number information.
0x081ab8f9 in __pthread_wait_for_restart_signal ()
(gdb)
Single stepping until exit from function __pthread_wait_for_restart_signal,
which has no line number information.
0x081a8c1c in pthread_cond_wait ()
(gdb)
Single stepping until exit from function pthread_cond_wait,
which has no line number information.
start_signal_handler () at mysqld.cc:1526
1526 pthread_mutex_unlock(&LOCK_thread_count);
(gdb)
1528 (void) pthread_attr_destroy(&thr_attr);
(gdb)
main (argc=37, argv=0x82ed478) at mysqld.cc:2076
2076 if (acl_init((THD*) 0, opt_noacl))
(gdb)
acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:141
141 if (!acl_cache)
(gdb)
144 (hash_free_key) free);
(gdb)
53 :size(size_arg), key_offset(key_offset_arg), key_length(key_length_arg),
(gdb)
56 bzero((char*) &cache,sizeof(cache));
(gdb)
144 (hash_free_key) free);
(gdb)
145 if (dont_read_acl_tables)
(gdb)
151 if (!(thd=new THD))
(gdb)
269 return (void*)my_malloc((uint)size, MYF(MY_WME | MY_FAE));
(gdb)
my_malloc (Size=2232, MyFlags=24) at my_malloc.c:33
33 if (!Size)
Current language: auto; currently c
(gdb)
35 if ((point = (char*)malloc(Size)) == NULL)
(gdb)
45 else if (MyFlags & MY_ZEROFILL)
(gdb)
48 DBUG_RETURN(point);
(gdb)
acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:151
151 if (!(thd=new THD))
Current language: auto; currently c++
(gdb)
THD::THD (this=0x82fefd0) at sql_list.h:278
278 prev=0; next=0;
(gdb)
85 {
(gdb) c
Continuing.
/usr/scratch/mysql-4.0/sql/mysqld: ready for connections

Breakpoint 1, 0x081c1857 in sigsuspend ()
(gdb)
Continuing.

ctrl+c, as the 'mysqladmin processlist' doesn't get any reply.


Program received signal SIGINT, Interrupt.
0x081c1869 in sigsuspend ()
(gdb) where
#0 0x081c1869 in sigsuspend ()
#1 0x081ab8f9 in __pthread_wait_for_restart_signal ()
#2 0x081ab283 in pthread_create ()
#3 0x08081df9 in create_new_thread (thd=0x82fefd0) at mysqld.cc:2500
#4 0x08082273 in handle_connections_sockets (arg=0x0) at mysqld.cc:2729
#5 0x08081a4d in main (argc=37, argv=0x82ed478) at mysqld.cc:2217
#6 0x081be865 in __libc_start_main ()
(gdb) detach
Detaching from program: /usr/scratch/mysql-4.0/sql/mysqld, process 4255
(gdb) quit
root@kulan:/scratch/mysql-4.0/sql# strace -v -v -v -f -p 4255
--- SIGCONT (Continued) ---
--- SIGSTOP (Stopped (signal)) ---
--- SIGCONT (Continued) ---

I tried to send CONT/STOP/CONT signal, no luck to rescue mysqld.
'kill -15' does not help, is it masked? So, I had to use 'kill -9'.
Does this make any sense to you?
Linux 2.4.20-rc2-ac1, debian


------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread13321@lists.mysql.com
To unsubscribe, e-mail

Re: crashing: /proc/613/exe: Permission denied

Hi,
would this help somehow?:


021120 10:45:34 Aborted connection 17842 to db: 'DIMA' user: 'frishman' host: `146.107.217.181' (Got an error reading communicat
ion packets)
021120 10:45:34 Aborted connection 17843 to db: 'Saccharomyces_cerevisiae' user: 'frishman' host: `146.107.217.181' (Got an erro
r reading communication packets)
mysqld: linuxthreads.c:64: linuxthreads_notify_others: Assertion `descr!=0' failed.

Number of processes running now: 1
mysqld process hanging, pid 7935 - killed
021120 10:48:32 mysqld restarted
021120 10:48:32 Can't start server: Bind on TCP/IP port: Address already in use
021120 10:48:32 Do you already have another mysqld server running on port: 3306 ?
021120 10:48:32 Aborting

021120 10:48:32 /usr/local/mysql/libexec/mysqld: Shutdown Complete

021120 10:48:32 mysqld ended

021120 10:49:28 mysqld started
crashing: /proc/30207/exe: Permission denied
BFD load failed..
/usr/local/mysql/libexec/mysqld: ready for connections
021120 10:51:27 /usr/local/mysql/libexec/mysqld: Normal shutdown

021120 10:51:27 Aborted connection 10 to db: 'Arabidopsis_thaliana' user: 'frishman' host: `146.107.217.46' (Got timeout reading
communication packets)
021120 10:51:29 /usr/local/mysql/libexec/mysqld: Shutdown Complete

021120 10:51:29 mysqld ended

021120 11:49:08 mysqld started
crashing: /proc/362/exe: Permission denied
BFD load failed..
/usr/local/mysql/libexec/mysqld: ready for connections
021120 12:17:44 Aborted connection 6 to db: 'Streptococcus_pyogenes_MGAS315' user: 'pedant' host: `146.107.217.64' (Got an error
reading communication packets)
021120 12:18:30 Aborted connection 12 to db: 'Erwinia_carotovora_atroseptica' user: 'pedant' host: `146.107.217.64' (Got an erro
r reading communication packets)
i


Linux i686 2.4.20-rc2-ac1 #1 SMP
mysql-20021125BK



On Mon, 11 Nov 2002, Michael Widenius wrote:

>
> Hi!
>
> >>>>> "Martin" == Martin MOKREJ writes:
>
> Martin> On Sun, 20 Oct 2002, Michael Widenius wrote:
> Martin> Hi Michael,
>
> MW> Martin> How-To-Repeat:
> MW>
> MW> Martin> 021018 13:19:33 mysqld started
> MW> Martin> crashing: /proc/613/exe: Permission denied
> MW> Martin> BFD load failed..
>
>
>
> Martin> /scratch/mysql-4.0# gdb ./sql/mysqld
>
> Martin> start_signal_handler () at mysqld.cc:1498
> Martin> 1498 if ((error=pthread_create(&signal_thread,&thr_attr,signal_hand, 0)))
> Martin> Current language: auto; currently c++
> Martin> (gdb)
> Martin> 1504 (void) pthread_cond_wait(&COND_thread_count,&LOCK_thread_count);
> Martin> (gdb) crashing: /proc/21712/exe: Permission denied
> Martin> BFD load failed..
>
> The above means that either something is wrong with your
> pthread_cond_wait() call or that because of some strange reason
> the signal handler died.
>
> You could try to verify this by setting a break point at 'signal_hand'
> and single step through this code.
>
>
>
> MW> Martin> BTW: Inspecting the trace log tells me, there are a lost of PATH's
> MW> Martin> hardcoded into binaries, and very often it tries to open ./anything before
> MW> Martin> using $BINDIR/anything or using similar variable set by mysqld_safe. Is
> MW> Martin> that expected?
> MW>
> MW> Can you give us some examples of this ?
> MW> Especially when searching after my.cnf files mysqld will try some
> MW> pre-determinate paths, but appart from that this shouldn't happen.
>
> Martin> OK, something a bit different what I meant previously, but let's think of this:
>
> Martin> [pid 21262] open("./mysql/func.frm", O_RDONLY|O_LARGEFILE) = 7
> Martin> [pid 21262] read(7, "\376\1\7\t\3\0\0\20\1\0\0000\0\0\347\0\303\0\0\0\0\0\0"..., 64) = 64
> Martin> [pid 21262] _llseek(7, 64, [64], SEEK_SET) = 0
> Martin> [pid 21262] read(7, "//\0\0 \0\0", 7) = 7
> Martin> [pid 21262] _llseek(7, 4096, [4096], SEEK_SET) = 0
> Martin> [pid 21262] read(7, "\1\1\23\0\n\0\2@\0\1\1\200\2\0\0\0\0@\0\377PRIMARY\377"..., 30) = 30
> Martin> [pid 21262] pread(7, "\377 "..., 195, 4327) = 195
> Martin> [pid 21262] _llseek(7, 8192, [8192], SEEK_SET) = 0
> Martin> [pid 21262] read(7, "\301\1\0\20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"... , 288) = 288
> Martin> [pid 21262] read(7, "M\0\5\4\2\24) "..., 161) = 161
> Martin> [pid 21262] close(7) = 0
> Martin> [pid 21262] getcwd("/data/mysql2", 4096) = 13
>
> Martin> --------------------^ we do getcwd() foreach mysql.* table, although it seems we already are
> Martin> in $datadir.
>
> MySQL has to call 'getwcd()' to be able to resolve the name to an
> absolute path (needed for the internal locking handling).
>
> MySQL will cache the result from getwcd() for future calls, so this
> isn't that bad.
>
> Martin> [pid 21262] readlink("./mysql/func.MYD", 0xbfffe84c, 511) = -1 EINVAL (Invalid argument)
>
>
> Martin> ------------------------------------------------------------ ---^ ?
>
> The above is ok, as func.MYD is not a symbolic link.
>
>
>
> MW> Martin> But those are probably only of interrest until my.cnf is found.
> MW>
> MW> Actually you should not get any errors like the above if mysqld fines
> MW> a my.cnf file as my.cnf files are searched for first. After this
> MW> mysqld should use the paths from the my.cnf file instead of any
> MW> internal paths.
>
>
> Martin> open("/etc/my.cnf", O_RDONLY|O_LARGEFILE) = 3
> Martin> fstat64(3, {st_dev=makedev(8, 1), st_ino=104, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=16, st_size=5592, st_atime=2002/11/04-14:30:06, st_mtime=2002/08/30-14:56:28, st_ctime=2002/08/30-14:56:28}) = 0
> Martin> old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40000000
> Martin> read(3, "# Example mysql config file for "..., 4096) = 4096
> Martin> read(3, "ing Innobase tables\n#innodb_data"..., 4096) = 1496
> Martin> read(3, "", 4096) = 0
> Martin> close(3) = 0
> Martin> munmap(0x40000000, 4096) = 0
> Martin> open("/usr/local/mysql/var/my.cnf", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> Martin> open("/root/.my.cnf", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
>
> Martin> Shouldn't we be more paranoid at least about the file permission on these configfiles?
> Martin> We read just anything, even world writable and owned by nobody. ;)
>
> For the clients this is ok, as one can anyway change all values in the
> ~/my.cnf file.
>
> I don't see any reason to slow down things by adding extra checks for
> the defaults file; Most programs that read option files does the
> same.
>
> Martin> stat64("/usr/local/mysql-BK20021104/share/mysql/charsets/lat in5.conf", {st_dev=makedev(8, 4), st_ino=81346, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=50, st_blksize=4096, st_blocks=16, st_size=4545, st_atime=2002/11/04-14:21:59, st_mtime=2002/11/04-14:21:59, st_ctime=2002/11/04-14:21:59}) = 0
> Martin> sched_setscheduler(0x530e, 0, 0xbffffaf8) = -1 EINVAL (Invalid argument)
>
> Martin> ----------------------------------------------------^ ?
>
> The above is ok and just means that you didn't configure MySQL with
> support for the above language.
>
> As MySQL does however allow you to add this support later, by just
> copying the file in place, MySQL has to check this on startup to know
> if the language is supported or not.
>
> Regards,
> Monty
>
>

--
Martin Mokrejs ,
PGP5.0i key is at http://www.natur.cuni.cz/~mmokrejs
MIPS / Institute for Bioinformatics
GSF - National Research Center for Environment and Health
Ingolstaedter Landstrasse 1, D-85764 Neuherberg, Germany
tel.: +49-89-3187 3683 , fax: +49-89-3187 3585

------------------------------------------------------------ ---------
Before posting, please check:
http://www.mysql.com/manual.php (the manual)
http://lists.mysql.com/ (the list archive)

To request this thread, e-mail bugs-thread13417@lists.mysql.com
To unsubscribe, e-mail