remove python dump_native_symbols. it does not work to callback into python in e.g. atexit,

filter native symbols for line profiling

Author: planrich

src/_vmprof.c

@@ -150,23 +150,6 @@ static void cpyprof_code_dealloc(PyObject *co)
     Original_code_dealloc(co);
 }
 
-
-void dump_native_symbols(int fileno)
-{
-    PyObject * mod = NULL;
-
-    mod = PyImport_ImportModuleNoBlock("vmprof");
-    if (mod == NULL)
-        goto error;
-
-    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
-
-error:
-    Py_XDECREF(mod);
-}
-
-
-
 static PyObject *enable_vmprof(PyObject* self, PyObject *args)
 {
     int fd;

src/symboltable.c

@@ -243,8 +243,6 @@ int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno, char
     return 0;
 }
 
-#ifdef RPYTHON_VMPROF
-
 #define WORD_SIZE sizeof(long)
 #define ADDR_SIZE sizeof(void*)
 #define MAXLEN 1024
@@ -254,10 +252,12 @@ void _dump_native_symbol(int fileno, void * addr, char * sym, int linenumber, ch
     off_t pos_before;
     struct str {
         void * addr;
-        // NOTE windows 64, not supported yet
+        // NOTE windows, not supported yet would be a problem for 64 bit
+        // hint: alignment
         long size;
         char str[1024];
     } s;
+    fsync(fileno);
     pos_before = lseek(fileno, 0, SEEK_CUR);
     lseek(fileno, 0, SEEK_END);
 
@@ -286,7 +286,7 @@ int _skip_string(int fileno)
 {
     long chars;
     int count = read(fileno, &chars, sizeof(long));
-    LOG("reading string of %d chars\n", chars);
+    //LOG("reading string of %d chars\n", chars);
     if (count <= 0) {
         return 1;
     }
@@ -309,14 +309,14 @@ int _skip_header(int fileno, int * version, int * flags)
 long _read_word(int fileno)
 {
     long w;
-    read(fileno, &w, WORD_SIZE);
+    (void)read(fileno, &w, WORD_SIZE);
     return w;
 }
 
 void * _read_addr(int fileno)
 {
     void * a;
-    read(fileno, &a, ADDR_SIZE);
+    (void)read(fileno, &a, ADDR_SIZE);
     return a;
 }
 
@@ -341,21 +341,19 @@ int _skip_time_and_zone(int fileno)
 
 void dump_native_symbols(int fileno)
 {
-    // only call this function
     off_t orig_pos, cur_pos;
     char marker;
     ssize_t count;
     int version;
     int flags;
     int memory = 0, lines = 0, native = 0;
+    fsync(fileno);
     orig_pos = lseek(fileno, 0, SEEK_CUR);
 
     lseek(fileno, 5*WORD_SIZE, SEEK_SET);
 
     while (1) {
-        LOG("pre read\n");
         count = read(fileno, &marker, 1);
-        LOG("post read\n");
         if (count <= 0) {
             break;
         }
@@ -383,7 +381,7 @@ void dump_native_symbols(int fileno)
                 break;
             } case MARKER_VIRTUAL_IP:
               case MARKER_NATIVE_SYMBOLS: {
-                LOG("virtip 0x%llx\n", cur_pos);
+                //LOG("virtip 0x%llx\n", cur_pos);
                 if (_skip_addr(fileno) != 0) { return; }
                 if (_skip_string(fileno) != 0) { return; }
                 break;
@@ -394,10 +392,16 @@ void dump_native_symbols(int fileno)
 
                 LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
 
+#ifdef RPYTHON_VMPROF
                 for (i = depth/2-1; i >= 0; i--) {
                     long kind = (long)_read_addr(fileno);
                     void * addr = _read_addr(fileno);
                     if (kind == VMPROF_NATIVE_TAG) {
+#else
+                for (i = 0; i < depth; i++) {
+                    void * addr = _read_addr(fileno);
+                    if (((intptr_t)addr & 0x1) == 1) {
+#endif
                         LOG("found kind %p\n", addr);
                         char name[MAXLEN];
                         char srcfile[MAXLEN];
@@ -412,14 +416,17 @@ void dump_native_symbols(int fileno)
                 }
                 LOG("passed  memory %d \n", memory);
 
-                if (_skip_addr(fileno) != 0) { return; } // thread id
+                if (version >= VERSION_THREAD_ID) {
+                    if (_skip_addr(fileno) != 0) { return; } // thread id
+                }
                 if (memory) {
                     if (_skip_addr(fileno) != 0) { return; } // profile memory
                 }
 
                 break;
             } default: {
                 fprintf(stderr, "unknown marker 0x%x\n", marker);
+                lseek(fileno, 0, SEEK_END);
                 return;
             }
         }
@@ -432,4 +439,3 @@ void dump_native_symbols(int fileno)
 
     lseek(fileno, 0, SEEK_END);
 }
-#endif

src/vmp_stack.c

@@ -126,7 +126,7 @@ static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, vo
 }
 
 int vmp_walk_and_record_python_stack_only(PY_STACK_FRAME_T *frame, void ** result,
-                                     int max_depth, int depth, intptr_t pc)
+                                          int max_depth, int depth, intptr_t pc)
 {
     while (depth < max_depth && frame) {
         frame = _write_python_stack_entry(frame, result, &depth, max_depth);

src/vmprof_main.h

@@ -343,7 +343,6 @@ void init_cpyprof(int native)
 static void disable_cpyprof(void)
 {
     vmp_native_disable();
-    dump_native_symbols(vmp_profile_fileno());
 }
 #endif
 
@@ -376,9 +375,15 @@ int vmprof_enable(int memory, int native)
 
 int close_profile(void)
 {
+    int fileno = vmp_profile_fileno();
+    fsync(fileno);
+    dump_native_symbols(fileno);
+
     (void)vmp_write_time_now(MARKER_TRAILER);
 
     teardown_rss();
+
+
     /* don't close() the file descriptor from here */
     vmp_set_profile_fileno(-1);
     return 0;

vmprof/__init__.py

@@ -73,40 +73,6 @@ def enable(fileno, period=DEFAULT_PERIOD, memory=False, lines=False, native=None
         native = _is_native_enabled(native)
         _vmprof.enable(gz_fileno, period, memory, lines, native)
 
-    def dump_native_symbols(fileno):
-        # native symbols cannot be resolved in the signal handler.
-        # it would take far too long. Thus this method should be called
-        # just after the sampling finished and before the file descriptor
-        # is closed.
-
-        # called from C with the fileno that has been used for this profile
-        # duplicates are avoided if this function is only called once for a profile
-        fileobj = io.open(fileno, mode='rb', closefd=False)
-        fileobj.seek(0)
-        _, profiles, _, _, _, _, _ = read_prof(fileobj)
-
-        duplicates = set()
-        fileobj = io.open(fileno, mode='ab', closefd=False)
-
-        for profile in profiles:
-            addrs = profile[0]
-            for addr in addrs:
-                if addr in duplicates:
-                    continue
-                duplicates.add(addr)
-                if addr & 0x1 and addr > 1:
-                    name, lineno, srcfile = _vmprof.resolve_addr(addr)
-                    if name == "" and srcfile == '-':
-                        name = "<native symbol 0x%x>" % addr
-
-                    str = "n:%s:%d:%s" % (name, lineno, srcfile)
-                    if PY3:
-                        str = str.encode()
-                    out = [MARKER_NATIVE_SYMBOLS, struct.pack("l", addr),
-                           struct.pack("l", len(str)),
-                           str]
-                    fileobj.write(b''.join(out))
-
     def sample_stack_now(skip=0):
         """ Helper utility mostly for tests, this is considered
             private API.

vmprof/show.py

@@ -193,11 +193,11 @@ def walk(node, d):
             parts = node.name.count(':')
             if parts == 3:
                 block_type, funname, funline, filename = node.name.split(':')
-
-                lines = d.setdefault((filename, int(funline), funname), {})
-
-                for l, cnt in six.iteritems(node.lines):
-                    lines[l] = lines.get(l, 0) + cnt
+                # only python supported for line profiling
+                if block_type == 'py':
+                    lines = d.setdefault((filename, int(funline), funname), {})
+                    for l, cnt in six.iteritems(node.lines):
+                        lines[l] = lines.get(l, 0) + cnt
 
             for c in six.itervalues(node.children):
                 walk(c, d)