[Rawstudio-users] Crash in rawstudio 1.2 when loading first photo in transform_nocms8_3dnow

[David Sharp]

The application starts, shows thumbnails across the top, but crashes
as soon as I open a photo.

It segfaults in transform_nocms8_3dnow, which is odd because I would
think it would be using the sse version, since I have an athlon64
which should support sse.

Here's a backtrace:

Program received signal SIGSEGV, Segmentation fault.
0x000000000045f88b in transform_nocms8_3dnow ()
Current language:  auto
The current source language is "auto; currently asm".
(gdb) bt
#0  0x000000000045f88b in transform_nocms8_3dnow ()
#1  0x0000000000461b2e in buffer ()
#2  0x0000000000461d0a in redraw ()
#3  0x0000000000463173 in rs_preview_widget_update ()
#4  0x0000000000464688 in rs_preview_widget_set_photo ()
#5  0x000000000041324c in rs_set_photo ()
#6  0x000000000042020f in icon_activated ()
#7  0x0000003cda20d12d in g_closure_invoke () from /usr/lib/libgobject-2.0.so.0
#8  0x0000003cda2222b2 in signal_emit_unlocked_R () from
/usr/lib/libgobject-2.0.so.0
#9  0x0000003cda223807 in g_signal_emit_valist () from
/usr/lib/libgobject-2.0.so.0
#10 0x0000003cda223cd3 in g_signal_emit () from /usr/lib/libgobject-2.0.so.0
#11 0x000000000045ed07 in selection_changed ()
#12 0x0000003cda20d12d in g_closure_invoke () from /usr/lib/libgobject-2.0.so.0
#13 0x0000003cda2222b2 in signal_emit_unlocked_R () from
/usr/lib/libgobject-2.0.so.0
#14 0x0000003cda223807 in g_signal_emit_valist () from
/usr/lib/libgobject-2.0.so.0
#15 0x0000003cda223cd3 in g_signal_emit () from /usr/lib/libgobject-2.0.so.0
#16 0x00007ffff7afe870 in gtk_icon_view_button_press () from
/usr/lib/libgtk-x11-2.0.so.0
#17 0x00007ffff7b22438 in _gtk_marshal_BOOLEAN__BOXED () from
/usr/lib/libgtk-x11-2.0.so.0
#18 0x0000003cda20d12d in g_closure_invoke () from /usr/lib/libgobject-2.0.so.0
#19 0x0000003cda221fa6 in signal_emit_unlocked_R () from
/usr/lib/libgobject-2.0.so.0
#20 0x0000003cda2236a0 in g_signal_emit_valist () from
/usr/lib/libgobject-2.0.so.0
#21 0x0000003cda223cd3 in g_signal_emit () from /usr/lib/libgobject-2.0.so.0
#22 0x00007ffff7c213a6 in gtk_widget_event_internal () from
/usr/lib/libgtk-x11-2.0.so.0
#23 0x00007ffff7b1b19b in gtk_propagate_event () from
/usr/lib/libgtk-x11-2.0.so.0
#24 0x00007ffff7b1c22b in gtk_main_do_event () from /usr/lib/libgtk-x11-2.0.so.0
#25 0x00007ffff779e0fc in gdk_event_dispatch () from
/usr/lib/libgdk-x11-2.0.so.0
#26 0x0000003cdae38e51 in g_main_context_dispatch () from
/usr/lib/libglib-2.0.so.0
#27 0x0000003cdae3c3e8 in g_main_context_iterate () from
/usr/lib/libglib-2.0.so.0
#28 0x0000003cdae3c8bd in g_main_loop_run () from /usr/lib/libglib-2.0.so.0
#29 0x00007ffff7b1c617 in gtk_main () from /usr/lib/libgtk-x11-2.0.so.0
#30 0x000000000041fe66 in gui_init ()
#31 0x0000000000412a19 in main ()


Here's some light debugging/disassembly work:

            d[destoffset++] = rct->table8[r];
            d[destoffset++] = rct->table8[g];
            d[destoffset++] = rct->table8[b];

0x000000000045f865 <transform_nocms8_3dnow+485>:    mov
0x148(%rdi),%rax       %rax = rct->table8
0x000000000045f86c <transform_nocms8_3dnow+492>:    movzbl (%rax,%rbx,1),%eax
0x000000000045f870 <transform_nocms8_3dnow+496>:    mov    %al,(%r9,%rdx,1)
0x000000000045f874 <transform_nocms8_3dnow+500>:    mov    0x148(%rdi),%rax
0x000000000045f87b <transform_nocms8_3dnow+507>:    movzbl (%rax,%rsi,1),%eax
0x000000000045f87f <transform_nocms8_3dnow+511>:    mov    %al,0x1(%r9,%rdx,1)
0x000000000045f884 <transform_nocms8_3dnow+516>:    mov
0x148(%rdi),%rax       %rax = rct->table8
0x000000000045f88b <transform_nocms8_3dnow+523>:    movzbl
(%rax,%r11,1),%eax     %eax = *(%rax+b)     *** crash ***
0x000000000045f890 <transform_nocms8_3dnow+528>:    mov
%al,0x2(%r9,%rdx,1)    d[destoffset+2] = %al
0x000000000045f895 <transform_nocms8_3dnow+533>:    add    $0x3,%rdx
           destoffset += 3

r => %rbx           = 0
g => %rsi           = 0
b => %r11           = 0x52f9000052f9    91229400355577
destoffset => %rdx  = 0
d => %r9            = 0xc89f80
mat => %rbp         = 0x7fffffffc920
*mat:
(gdb) x/12wf 0x7fffffffc920
0x7fffffffc920:	1.2081722	-0.15823926	-0.0499329716	0
0x7fffffffc930:	0.0230940226	1.23610961	-0.259203643	0
0x7fffffffc940:	0.0542779602	-0.188402981	1.13412499	0
(gdb) x/12wx 0x7fffffffc920
0x7fffffffc920:	0x3f9aa563	0xbe220979	0xbd4c8684	0x00000000
0x7fffffffc930:	0x3cbd2fad	0x3f9e38d7	0xbe84b657	0x00000000
0x7fffffffc940:	0x3d5e5291	0xbe40ecb6	0x3f912b02	0x00000000


As you can see, b is waaay off.  I think 0x529f is a more reasonable
value (the lower 32 bits)?