Computing time offsets between EXIF and GPS I like the idea of matching photos to GPS traces. In Debian there is gpscorrelate but it's almost unusable to me because of bug #473362 and it has an awkward way of specifying time offsets. Here at SoTM10 someone told me that exiftool gained -geosync and -geotag options. So it's just a matter of creating a little tool that shows a photo and asks you to type the GPS time you see in it. Apparently there are no bindings or GIR files for gtkimageview in Debian, so I'll have to use C. Here is a C prototype:

/*
 * gpsoffset - Compute EXIF time offset from a photo of a gps display
 *
 * Use with exiftool -geosync=... -geotag trace.gpx DIR
 *
 * Copyright (C) 2009--2010  Enrico Zini <enrico@enricozini.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
#define _XOPEN_SOURCE /* glibc2 needs this */
#include <time.h>
#include <gtkimageview/gtkimageview.h>
#include <libexif/exif-data.h>
#include <stdio.h>
#include <stdlib.h>
static int load_time(const char* fname, struct tm* tm)
 
    ExifData* exif_data = exif_data_new_from_file(fname);
    ExifEntry* exif_time = exif_data_get_entry(exif_data, EXIF_TAG_DATE_TIME);
    if (exif_time == NULL)
     
        fprintf(stderr, "Cannot find EXIF timetamp\n");
        return -1;
     
    char buf[1024];
    exif_entry_get_value(exif_time, buf, 1024);
    //printf("val2: %s\n", exif_entry_get_value(t2, buf, 1024));
    if (strptime(buf, "%Y:%m:%d %H:%M:%S", tm) == NULL)
     
        fprintf(stderr, "Cannot match EXIF timetamp\n");
        return -1;
     
    return 0;
 
static time_t exif_ts;
static GtkWidget* res_lbl;
void date_entry_changed(GtkEditable *editable, gpointer user_data)
 
    const gchar* text = gtk_entry_get_text(GTK_ENTRY(editable));
    struct tm parsed;
    if (strptime(text, "%Y-%m-%d %H:%M:%S", &parsed) == NULL)
     
        gtk_label_set_text(GTK_LABEL(res_lbl), "Please enter a date as YYYY-MM-DD HH:MM:SS");
      else  
        time_t img_ts = mktime(&parsed);
        int c;
        int res;
        if (exif_ts < img_ts)
         
            c = '+';
            res = img_ts - exif_ts;
         
        else
         
            c = '-';
            res = exif_ts - img_ts;
         
        char buf[1024];
        if (res > 3600)
            snprintf(buf, 1024, "Result: %c%ds -geosync=%c%d:%02d:%02d",
                    c, res, c, res / 3600, (res / 60) % 60, res % 60);
        else if (res > 60)
            snprintf(buf, 1024, "Result: %c%ds -geosync=%c%02d:%02d",
                    c, res, c, (res / 60) % 60, res % 60);
        else
            snprintf(buf, 1024, "Result: %c%ds -geosync=%c%d",
                    c, res, c, res);
        gtk_label_set_text(GTK_LABEL(res_lbl), buf);
     
 
int main (int argc, char *argv[])
 
    // Work in UTC to avoid mktime applying DST or timezones
    setenv("TZ", "UTC");
    const char* filename = "/home/enrico/web-eddie/galleries/2010/04-05-Uppermill/P1080932.jpg";
    gtk_init (&argc, &argv);
    struct tm exif_time;
    if (load_time(filename, &exif_time) != 0)
        return 1;
    printf("EXIF time: %s\n", asctime(&exif_time));
    exif_ts = mktime(&exif_time);
    GtkWidget* window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
    GtkWidget* vb = gtk_vbox_new(FALSE, 0);
    GtkWidget* hb = gtk_hbox_new(FALSE, 0);
    GtkWidget* lbl = gtk_label_new("Timestamp:");
    GtkWidget* exif_lbl;
     
        char buf[1024];
        strftime(buf, 1024, "EXIF time: %Y-%m-%d %H:%M:%S", &exif_time);
        exif_lbl = gtk_label_new(buf);
     
    GtkWidget* date_ent = gtk_entry_new();
    res_lbl = gtk_label_new("Result:");
    GtkWidget* view = gtk_image_view_new();
    GdkPixbuf* pixbuf = gdk_pixbuf_new_from_file(filename, NULL);
    gtk_box_pack_start(GTK_BOX(hb), lbl, FALSE, TRUE, 0);
    gtk_box_pack_start(GTK_BOX(hb), date_ent, TRUE, TRUE, 0);
    gtk_signal_connect(GTK_OBJECT(date_ent), "changed", (GCallback)date_entry_changed, NULL);
     
        char buf[1024];
        strftime(buf, 1024, "%Y-%m-%d %H:%M:%S", &exif_time);
        gtk_entry_set_text(GTK_ENTRY(date_ent), buf);
     
    gtk_widget_set_size_request(view, 500, 400);
    gtk_image_view_set_pixbuf(GTK_IMAGE_VIEW(view), pixbuf, TRUE);
    gtk_container_add(GTK_CONTAINER(window), vb);
    gtk_box_pack_start(GTK_BOX(vb), view, TRUE, TRUE, 0);
    gtk_box_pack_start(GTK_BOX(vb), hb, FALSE, TRUE, 0);
    gtk_box_pack_start(GTK_BOX(vb), exif_lbl, FALSE, TRUE, 0);
    gtk_box_pack_start(GTK_BOX(vb), res_lbl, FALSE, TRUE, 0);
    gtk_widget_show_all(window);
    gtk_main ();
    return 0;
 

And here is its simple makefile:

CFLAGS=$(shell pkg-config --cflags gtkimageview libexif)
LDFLAGS=$(shell pkg-config --libs gtkimageview libexif)
gpsoffset: gpsoffset.c

It's a simple prototype but it's a working prototype and seems to do the job for me. I currently cannot find out why after I click on the text box, there seems to be no way to give the focus back to the image viewer so I can control it with keys. There is another nice algorithm to compute time offsets to be implemented: you choose a photo taken from a known place and drag it on that place on a map: you can then look for the nearest point on your GPX trace and compute the time offset from that. I have seen that there are programs for geotagging photos that implement all such algorithms, and have a nice UI, but I haven't seen any in Debian. Are there any such softwares that can be packaged? If not, the interpolation and annotation tasks can now already be performed by exiftool, so it's just a matter of building a good UI, and I would love to see someone picking up the task.

Searching OSM nodes in Spatialite Third step of my SoTM10 pet project: finding the POIs. I put together a query to find all nodes with a given tag inside a bounding box, and also a query to find all the tag values for a given tag name inside a bounding box. The result is this simple POI search engine:

#
# poisearch - simple geographical POI search engine
#
# Copyright (C) 2010  Enrico Zini <enrico@enricozini.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
from pysqlite2 import dbapi2 as sqlite
class PoiDB(object):
    def __init__(self):
        self.db = sqlite.connect("pois.db")
        self.db.enable_load_extension(True)
        self.db.execute("SELECT load_extension('libspatialite.so')")
        self.oldsearch = []
        self.bbox = None
    def set_bbox(self, xmin, xmax, ymin, ymax):
        '''Set bbox for searches'''
        self.bbox = (xmin, xmax, ymin, ymax)
    def tagid(self, name, val):
        '''Get the database ID for a tag'''
        c = self.db.cursor()
        c.execute("SELECT id FROM tag WHERE name=? AND value=?", (name, val))
        res = None
        for row in c:
            res = row[0]
        return res
    def tagnames(self):
        '''Get all tag names'''
        c = self.db.cursor()
        c.execute("SELECT DISTINCT name FROM tag ORDER BY name")
        for row in c:
            yield row[0]
    def tagvalues(self, name, use_bbox=False):
        '''
        Get all tag values for a given tag name,
        optionally in the current bounding box
        '''
        c = self.db.cursor()
        if self.bbox is None or not use_bbox:
            c.execute("SELECT DISTINCT value FROM tag WHERE name=? ORDER BY value", (name,))
        else:
            c.execute("SELECT DISTINCT tag.value FROM poi, poitag, tag"
                      " WHERE poi.rowid IN (SELECT pkid FROM idx_poi_geom WHERE ("
                      "       xmin >= ? AND xmax <= ? AND ymin >= ? AND ymax <= ?) )"
                      "   AND poitag.tag = tag.id AND poitag.poi = poi.id"
                      "   AND tag.name=?",
                      self.bbox + (name,))
        for row in c:
            yield row[0]
    def search(self, name, val):
        '''Get all name:val tags in the current bounding box'''
        # First resolve the tagid
        tagid = self.tagid(name, val)
        if tagid is None: return
        c = self.db.cursor()
        c.execute("SELECT poi.name, poi.data, X(poi.geom), Y(poi.geom) FROM poi, poitag"
                  " WHERE poi.rowid IN (SELECT pkid FROM idx_poi_geom WHERE ("
                  "       xmin >= ? AND xmax <= ? AND ymin >= ? AND ymax <= ?) )"
                  "   AND poitag.tag = ? AND poitag.poi = poi.id",
                  self.bbox + (tagid,))
        self.oldsearch = []
        for row in c:
            self.oldsearch.append(row)
            yield row[0], simplejson.loads(row[1]), row[2], row[3]
    def count(self, name, val):
        '''Count all name:val tags in the current bounding box'''
        # First resolve the tagid
        tagid = self.tagid(name, val)
        if tagid is None: return
        c = self.db.cursor()
        c.execute("SELECT COUNT(*) FROM poi, poitag"
                  " WHERE poi.rowid IN (SELECT pkid FROM idx_poi_geom WHERE ("
                  "       xmin >= ? AND xmax <= ? AND ymin >= ? AND ymax <= ?) )"
                  "   AND poitag.tag = ? AND poitag.poi = poi.id",
                  self.bbox + (tagid,))
        for row in c:
            return row[0]
    def replay(self):
        for row in self.oldsearch:
            yield row[0], simplejson.loads(row[1]), row[2], row[3]

Problem 3 solved: now on to the next step, building a user interface for it.

Importing OSM nodes into Spatialite Second step of my SoTM10 pet project: creating a searchable database with the points. What a fantastic opportunity to learn Spatialite. Learning Spatialite is easy. For example, you can use the two tutorials with catchy titles that assume your best wish in life is to create databases out of shapefiles using a pre-built, i386-only executable GUI binary downloaded over an insecure HTTP connection. To be fair, the second of those tutorials is called "An almost Idiot's Guide", thus expliciting the requirement of being an almost idiot in order to happily acquire and run software in that way. Alternatively, you can use A quick tutorial to SpatiaLite which is so quick it has examples that lead you to write SQL queries that trigger all sorts of vague exceptions at insert time. But at least it brought me a long way forward, at which point I could just cross reference things with PostGIS documentation to find out the right way of doing things. So, here's the importer script, which will probably become my reference example for how to get started with Spatialite, and how to use Spatialite from Python:

#!/usr/bin/python
#
# poiimport - import nodes from OSM into a spatialite DB
#
# Copyright (C) 2010  Enrico Zini <enrico@enricozini.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
import xml.sax
import xml.sax.handler
from pysqlite2 import dbapi2 as sqlite
import simplejson
import sys
import os
class OSMPOIReader(xml.sax.handler.ContentHandler):
    '''
    Filter SAX events in a OSM XML file to keep only nodes with names
    '''
    def __init__(self, consumer):
        self.consumer = consumer
    def startElement(self, name, attrs):
        if name == "node":
            self.attrs = attrs
            self.tags = dict()
        elif name == "tag":
            self.tags[attrs["k"]] = attrs["v"]
    def endElement(self, name):
        if name == "node":
            lat = float(self.attrs["lat"])
            lon = float(self.attrs["lon"])
            id = int(self.attrs["id"])
            #dt = parse(self.attrs["timestamp"])
            uid = self.attrs.get("uid", None)
            uid = int(uid) if uid is not None else None
            user = self.attrs.get("user", None)
            self.consumer(lat, lon, id, self.tags, user=user, uid=uid)
class Importer(object):
    '''
    Create the spatialite database and populate it
    '''
    TAG_WHITELIST = set(["amenity", "shop", "tourism", "place"])
    def __init__(self, filename):
        self.db = sqlite.connect(filename)
        self.db.enable_load_extension(True)
        self.db.execute("SELECT load_extension('libspatialite.so')")
        self.db.execute("SELECT InitSpatialMetaData()")
        self.db.execute("INSERT INTO spatial_ref_sys (srid, auth_name, auth_srid,"
                        " ref_sys_name, proj4text) VALUES (4326, 'epsg', 4326,"
                        " 'WGS 84', '+proj=longlat +ellps=WGS84 +datum=WGS84"
                        " +no_defs')")
        self.db.execute("CREATE TABLE poi (id int not null unique primary key,"
                        " name char, data text)")
        self.db.execute("SELECT AddGeometryColumn('poi', 'geom', 4326, 'POINT', 2)")
        self.db.execute("SELECT CreateSpatialIndex('poi', 'geom')")
        self.db.execute("CREATE TABLE tag (id integer primary key autoincrement,"
                        " name char, value char)")
        self.db.execute("CREATE UNIQUE INDEX tagidx ON tag (name, value)")
        self.db.execute("CREATE TABLE poitag (poi int not null, tag int not null)")
        self.db.execute("CREATE UNIQUE INDEX poitagidx ON poitag (poi, tag)")
        self.tagid_cache = dict()
    def tagid(self, k, v):
        key = (k, v)
        res = self.tagid_cache.get(key, None)
        if res is None:
            c = self.db.cursor()
            c.execute("SELECT id FROM tag WHERE name=? AND value=?", key)
            for row in c:
                self.tagid_cache[key] = row[0]
                return row[0]
            self.db.execute("INSERT INTO tag (id, name, value) VALUES (NULL, ?, ?)", key)
            c.execute("SELECT last_insert_rowid()")
            for row in c:
                res = row[0]
            self.tagid_cache[key] = res
        return res
    def __call__(self, lat, lon, id, tags, user=None, uid=None):
        # Acquire tag IDs
        tagids = []
        for k, v in tags.iteritems():
            if k not in self.TAG_WHITELIST: continue
            for val in v.split(";"):
                tagids.append(self.tagid(k, val))
        # Skip elements that don't have the tags we want
        if not tagids: return
        geom = "POINT(%f %f)" % (lon, lat)
        self.db.execute("INSERT INTO poi (id, geom, name, data)"
                        "     VALUES (?, GeomFromText(?, 4326), ?, ?)",
                (id, geom, tags["name"], simplejson.dumps(tags)))
        for tid in tagids:
            self.db.execute("INSERT INTO poitag (poi, tag) VALUES (?, ?)", (id, tid))
    def done(self):
        self.db.commit()
# Get the output file name
filename = sys.argv[1]
# Ensure we start from scratch
if os.path.exists(filename):
    print >>sys.stderr, filename, "already exists"
    sys.exit(1)
# Import
parser = xml.sax.make_parser()
importer = Importer(filename)
handler = OSMPOIReader(importer)
parser.setContentHandler(handler)
parser.parse(sys.stdin)
importer.done()

Let's run it:

$ ./poiimport pois.db < pois.osm 
SpatiaLite version ..: 2.4.0    Supported Extensions:
        - 'VirtualShape'        [direct Shapefile access]
        - 'VirtualDbf'          [direct Dbf access]
        - 'VirtualText'         [direct CSV/TXT access]
        - 'VirtualNetwork'      [Dijkstra shortest path]
        - 'RTree'               [Spatial Index - R*Tree]
        - 'MbrCache'            [Spatial Index - MBR cache]
        - 'VirtualFDO'          [FDO-OGR interoperability]
        - 'SpatiaLite'          [Spatial SQL - OGC]
PROJ.4 Rel. 4.7.1, 23 September 2009
GEOS version 3.2.0-CAPI-1.6.0
$ ls -l --si pois*
-rw-r--r-- 1 enrico enrico 17M Jul  9 23:44 pois.db
-rw-r--r-- 1 enrico enrico 37M Jul  9 16:20 pois.osm
$ spatialite pois.db
SpatiaLite version ..: 2.4.0    Supported Extensions:
        - 'VirtualShape'        [direct Shapefile access]
        - 'VirtualDbf'          [direct DBF access]
        - 'VirtualText'         [direct CSV/TXT access]
        - 'VirtualNetwork'      [Dijkstra shortest path]
        - 'RTree'               [Spatial Index - R*Tree]
        - 'MbrCache'            [Spatial Index - MBR cache]
        - 'VirtualFDO'          [FDO-OGR interoperability]
        - 'SpatiaLite'          [Spatial SQL - OGC]
PROJ.4 version ......: Rel. 4.7.1, 23 September 2009
GEOS version ........: 3.2.0-CAPI-1.6.0
SQLite version ......: 3.6.23.1
Enter ".help" for instructions
spatialite> select id from tag where name="amenity" and value="fountain";
24
spatialite> SELECT poi.name, poi.data, X(poi.geom), Y(poi.geom) FROM poi, poitag WHERE poi.rowid IN (SELECT pkid FROM idx_poi_geom WHERE (xmin >= 2.56 AND xmax <= 2.90 AND ymin >= 41.84 AND ymax <= 42.00) ) AND poitag.tag = 24 AND poitag.poi = poi.id;
Font Picant de la Cellera "amenity": "fountain", "name": "Font Picant de la Cellera" 2.616045 41.952449
Font de Can Pla "amenity": "fountain", "name": "Font de Can Pla" 2.622354 41.974724
Font de Can Ribes "amenity": "fountain", "name": "Font de Can Ribes" 2.62311 41.979193

It's impressive: I've got all sort of useful information for the whole of Spain in just 17Mb! Let's put it to practice: I'm thirsty, is there any water fountain nearby?

spatialite> SELECT count(1) FROM poi, poitag WHERE poi.rowid IN (SELECT pkid FROM idx_poi_geom WHERE (xmin >= 2.80 AND xmax <= 2.85 AND ymin >= 41.97 AND ymax <= 42.00) ) AND poitag.tag = 24 AND poitag.poi = poi.id;
0

Ouch! No water fountains mapped in Girona... yet. Problem 2 solved: now on to the next step, trying to show the results in some usable way.

Filtering nodes out of OSM files I have a pet project here at SoTM10: create a tool for searching nearby POIs while offline. The idea is to have something in my pocket (FreeRunner or N900), which doesn't require an internet connection, and which can point me at the nearest fountains, post offices, atm machines, bars and so on. The first step is to obtain a list of POIs. In theory one can use Xapi but all the known Xapi servers appear to be down at the moment. Another attempt is to obtain it by filtering all nodes with the tags we want out of a planet OSM extract. I downloaded the Spanish one and set to work. First I tried with xmlstarlet, but it ate all the RAM and crashed my laptop, because for some reason, on my laptop the Linux kernels up to 2.6.32 (don't now about later ones) like to swap out ALL running apps to cache I/O operations, which mean that heavy I/O operations swap out the very programs performing them, so the system gets caught in some infinite I/O loop and dies. Or at least this is what I've figured out so far. So, we need SAX. I put together this prototype in Python, which can process a nice 8MB/s of OSM data for quite some time with a constant, low RAM usage:

#!/usr/bin/python
#
# poifilter - extract interesting nodes from OSM XML files
#
# Copyright (C) 2010  Enrico Zini <enrico@enricozini.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
import xml.sax
import xml.sax.handler
import xml.sax.saxutils
import sys
class XMLSAXFilter(xml.sax.handler.ContentHandler):
    '''
    A SAX filter that is a ContentHandler.

    There is xml.sax.saxutils.XMLFilterBase in the standard library but it is
    undocumented, and most of the examples using it you find online are wrong.
    You can look at its source code, and at that point you find out that it is
    an offensive practical joke.
    '''
    def __init__(self, downstream):
        self.downstream = downstream
    # ContentHandler methods
    def setDocumentLocator(self, locator):
        self.downstream.setDocumentLocator(locator)
    def startDocument(self):
        self.downstream.startDocument()
    def endDocument(self):
        self.downstream.endDocument()
    def startPrefixMapping(self, prefix, uri):
        self.downstream.startPrefixMapping(prefix, uri)
    def endPrefixMapping(self, prefix):
        self.downstream.endPrefixMapping(prefix)
    def startElement(self, name, attrs):
        self.downstream.startElement(name, attrs)
    def endElement(self, name):
        self.downstream.endElement(name)
    def startElementNS(self, name, qname, attrs):
        self.downstream.startElementNS(name, qname, attrs)
    def endElementNS(self, name, qname):
        self.downstream.endElementNS(name, qname)
    def characters(self, content):
        self.downstream.characters(content)
    def ignorableWhitespace(self, chars):
        self.downstream.ignorableWhitespace(chars)
    def processingInstruction(self, target, data):
        self.downstream.processingInstruction(target, data)
    def skippedEntity(self, name):
        self.downstream.skippedEntity(name)
class OSMPOIHandler(XMLSAXFilter):
    '''
    Filter SAX events in a OSM XML file to keep only nodes with names
    '''
    PASSTHROUGH = ["osm", "bound"]
    TAG_WHITELIST = set(["amenity", "shop", "tourism", "place"])
    def startElement(self, name, attrs):
        if name in self.PASSTHROUGH:
            self.downstream.startElement(name, attrs)
        elif name == "node":
            self.attrs = attrs
            self.tags = []
            self.propagate = False
        elif name == "tag":
            if self.tags is not None:
                self.tags.append(attrs)
                if attrs["k"] in self.TAG_WHITELIST:
                    self.propagate = True
        else:
            self.tags = None
            self.attrs = None
    def endElement(self, name):
        if name in self.PASSTHROUGH:
            self.downstream.endElement(name)
        elif name == "node":
            if self.propagate:
                self.downstream.startElement("node", self.attrs)
                for attrs in self.tags:
                    self.downstream.startElement("tag", attrs)
                    self.downstream.endElement("tag")
                self.downstream.endElement("node")
    def ignorableWhitespace(self, chars):
        pass
    def characters(self, content):
        pass
# Simple stdin->stdout XMl filter
parser = xml.sax.make_parser()
handler = OSMPOIHandler(xml.sax.saxutils.XMLGenerator(sys.stdout, "utf-8"))
parser.setContentHandler(handler)
parser.parse(sys.stdin)

Let's run it:

$ bzcat /store/osm/spain.osm.bz2   pv   ./poifilter > pois.osm
[...]
$ ls -l --si pois.osm
-rw-r--r-- 1 enrico enrico 19M Jul 10 23:56 pois.osm
$ xmlstarlet val pois.osm 
pois.osm - valid

Problem 1 solved: now on to the next step: importing the nodes in a database.

Request Tracker (also know as RT), the defacto Open Source ticket tracking software, has a special status value to get rid of spam tickets: deleted. It does exactly what it's supposed to do, that is get rid of these tickets quickly.

Unfortunately, it's very easy to accidentally delete a bunch of non-spam tickets via the bulk update interface. Restoring these deleted tickets is not straightforward.

Good news: none of the lost tickets were removed from the database.
Bad news: you can't search for deleted tickets.

Finding the lost ticketsIf you have a copy of the page before you did the bulk update (perhaps in your browser cache?) then you've got all you need. Otherwise, you'll need to access the database directly to get a list of ticket IDs.

First of all, find the user ID of the person who marked of all these tickets as deleted (42 in this example), then issue the following query:

 SELECT id, lastupdated, subject
 FROM tickets
WHERE status = 'deleted' AND
      lastupdatedby = 42
ORDER BY lastupdated;

You may want to filter some more using the lastupdated timestamp if you know roughly when the deletion occurred.

Restoring deleted ticketsSince all of the tickets are still in the database, all you need to do is to go to each of them directly (e.g. http://www.example.com/Ticket/Display.html?id=12345 for Ticket #12345) and change the status back to "open".

Request Tracker is a really neat support tool, but one of the common complaints I heard from people using it during a previous project was that it was pretty slow.

There wasn't much we could do about the (overloaded) server it was running on, but I found that enabling mod_deflate really helped.

After watching this great video though, I was inspired to look into it a bit more, focussing this time on latency.

Description of tests

• RT 3.6.7-5+lenny4
• Running on a Debian Lenny vserser.
• Server is on the LAN.
• Firefox 3.6.6 client (with Firebug 1.5.0)

Also note that I was looking for the "best case" for each of the different configurations and so each screenshot was taken after reloading the homepage 10-20 times to maximize cache hits (thanks in large part to mod_expires).

(Is there a nice automated way of measuring average latency?)

Stock RT 3.6Using the default apache2-modperl2 config file (as supplied by RT), here's what the homepage (logged in as root) looked like before I changed anything:



The purple section here indicates the time spent waiting for the server. This shows that the server (running Mason inside mod_perl) is doing quite a bit of processing, including a lot more work than you'd expect while serving static files. It's quite impressive to see how fast the images are being served (directly by Apache) in comparison with the Javascript and the CSS files (which go through Mason).

The reason while Javascript and CSS files have to be served by mod_perl is that they are in fact templates. They contain a few Mason variables which must be substituted before being served.

Looking into it further though, all of these replacements have to do with variables defined in RT_SiteConfig.pm (mostly the install path). Here's an example:

var path = "" ? "" : "/";

which gets turned into:

var path = "/rt" ? "/rt" : "/";

So as long as these paths don't change, then there is no need to re-generate these files.

Static Javascript and CSSThis next diagram was produced after configuring Apache to serve all Javascript and CSS files directly from Apache:



The way I did that (without modifying any of the original files) was by saving the Javascript/CSS sent to the browser and using mod_rewrite rules to serve these files instead of the original templated ones:

# Serve static files directly
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/ahah.js$ /var/www/rt/ahah.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/cascaded.js$ /var/www/rt/cascaded.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/class.js$ /var/www/rt/class.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/combobox.js$ /var/www/rt/combobox.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/list.js$ /var/www/rt/list.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/titlebox-state.js$ /var/www/rt/titlebox-state.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/js/util.js$ /var/www/rt/util.js
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/css/3.5-default/main-squished.css$ /var/www/rt/main-squished.css
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/css/print.css$ /var/www/rt/print.css
RewriteRule ^/usr/share/request-tracker3.6/html/NoAuth/webrtfm.css$ /var/www/rt/webrtfm.css

Removing unnecessary images

Finally, one thing I noticed from this last graph is that the rounded corners in the theme require a number of small images. While these don't take a whole lot of bandwidth, they do require quite a bit of back and forth between the browser and the server.

So I replaced all of the "rounded corner" images in the main-squished.css file with the following CSS attributes:

-moz-border-radius-topleft: 8px;
-moz-border-radius-topright: 8px;
-moz-border-radius-bottomleft: 8px;
-moz-border-radius-bottomright: 8px;
-webkit-border-top-left-radius: 8px;
-webkit-border-top-right-radius: 8px;
-webkit-border-bottom-left-radius: 8px;
-webkit-border-bottom-right-radius: 8px;

(yes, Internet Explorer users probably don't get the rounded corners... oh well)

This eliminated a number of roundtrips and shaved off a few more milliseconds:



Merging CSS and Javascript filesBy this stage, the pages are pretty snappy so there is not much to be gained anymore, but I figured I'd try to reduce the latency a bit more by combining all Javascript files into one (and doing the same for CSS files with the exception of print.css). This is what I got:



(Note that I also took the opportunity to minify both squished files to reduce the filesize.)

Not a huge improvement and I unfortunately had to copy quite a few Mason templates from /usr/share/request-tracker3.6/html/ to /usr/local/share/request-tracker3.6/html/ and then replace all of the script tags with a single one in html/Elements/Header.

Others things to look intoI've stopped here, but there might be ways to further reduce the processing time on the server (hence the latency) by tuning mod_perl/Mason or Postgres. The RT wiki also has a few pointers.

Replacing Apache with Nginx (which means moving to FastCGI) was something I considered, but after trying it out, it turned out that it would add about 100ms of extra latency.

Feel free to leave a comment if you've found something else that makes a big difference on your site.

Bug-Squashing-Party in Munich We are organizing another BSP in Munich on the weekend of 17th/18th July. Like the last BSP, it will take place in the LiMux office in the city center. See the coordination page for further information and directions and in order to sign up for it. Food and drinks for the event are partly sponsored by the City of Munich and this amount (100 ) has been again matched by credativ GmbH. Contributions by non-DDs are welcome as always; the BSP will be attended by several Debian Developers who will be able to upload fixes. Attending BSPs is a great way to get involved in the Debian community! We cannot guarantee crash space to people from outside Munich at this point, so if you want to attend please contact the Munich list (area-muc@teams.debian.net) beforehand or arrange your own accomodation. We probably start the BSP at some point after 6PM on Friday evening already, but the main action will be on Saturday and Sunday. As usual, people should bring their notebooks and possibly an ethernet cable. Wireless will be present as well, but a certain bandwidth cannot be guaranteed. Bug-Squashing Introduction I also plan to give an introduction to bug-squashing at this month's Garching Debian Stammtisch, which will take place next Wednesday, July 14th in the campus-cneipe (or its beergarden) as usual (the monthly Stammtisch is always on the second Wednesday of the month). If you study/work in Garching and would like to attend the BSP, come to the Stammtisch for a tutorial (and beer)! Debian Packaging Tutorial Some time ago, I gave a tutorial on Debian packaging in the TechTalk series of the Open-Source- School. The audience seemed to be pleased with the talk, and as more people requested to attend than there were seats in the room, we decided to reprise the event on September 7th. Attendence is free, but you have to register (see the event page). There are still some places left at this point.

Currently I own and use an iPhone 3G. I bought it almost two years ago, when the local phone provider LMT started offering the iPhone legally. I had a pretty good experience with it most of the time, but now it is showing its age:

• The two year warranty will run out in September
• The iOS 4 update left out most of the new features my model cann t have multitasking or backgrounds or any of that new cool stuff that is exclusive to the new iPhone 4
• The hardware itself is starting to wear the speaker stopped working a few weeks ago (so I have to use the loudspeaker function or headphones) and also approximately around that time the phone stated to randomly lock up approximately one a week or two it freezes and after a few minutes reboots and demads to be connected to iTunes and restored from backup
• Also the battery does not last as long as it used to

I will try to send my iPhone in for repair hoping to extend its usefulness, but frankly it looks that I might have to get a new phone at or before Debconf. So I am considering my options for a new phone and so far I have 3 main options each with some sub-options. The main options are: iPhone 4, MeeGo and Android. Each choice has its benefits and drawbacks and each also has several sub-options related either to specific phone models or to purchase methods. What I need is a phone with: calling function, contact information synchronised to Google, support for Google Mail/Contacts/Calendar simultaneously with one Exchange account also providing mail/contacts/calendars, music, audiobooks saving my place in the audio book and in the list of books, fully featured Twitter client with Twitpic and location support, encrypted password storage application (preferably open source), GPS with maps, ability to download maps for offline use, Geocaching support with offline cache info and logging, Skype client, ability to write in Latvian, English and Russian in all applications, reliable backup and restore of phones data including settings and data of installed apps. Optional features: last.fm streaming and reporting support, ebook applications with purchasable ebooks, high quality video and photo recording (for a phone), ability for me to write applications on the phone, ssh client, wireless data upload to the phone, ability to change the phones battery without loosing data in the phone (on the go), compass, physical keyboard, confidence in getting software upgrades and developer interest for the next 2 years. I ll try to describe all of the options here to organise my thoughts and maybe also help someone else make the choice. Option 1: iPhone 4 To get the iPhone 4 I have two options: LMT and unlocked. The official arrival of iPhone 4 to Latvia is expected in September and it will likely be sold by LMT for the same prices as iPhone 3GS is now. If I choose this scenario, I will need to survive with my semi-broken iPhone for 2-3 more months. Total price over 2 years (LVL): 131 (purchase price) + 24 * 8 (surcharge) = 323 LVL. Note: the surcharge is the extra cost of the iPhone call plan compared to a comparable regular plan (Vien dais 7 + Internets telefon 5). If I go and purchase an unlocked iPhone 4 in a country where such units are sold (UK is the most easily accessible choice, because a few of my workmates travel weekly to and from UK) then the total price over 2 years would be the purchase price: 499 GBP + 65 GBP (2 year warranty extension) = 505 LVL Now lets go over the general benefits and downsides of the iPhone 4, first the positives:

• It is a simple to use device that I am used to over the two years
• The screen resolution is higher than any other option
• It is smaller and lighter than Nokia N900 and the bigger Android devices
• There is a huge number of free and paid apps as well as accessories for the iPhone docks, headphones, chargers,
• I know that it can do every thing that I need it to do (with apps) and most of the optional features too, except (see drawbacks)

Drawbacks:

• The first thing that I hate about the iPhone is its umbilical cord like link to iTunes. iTunes is the sole purpose I still have a real Windows installation on my main computer (in a Sun VirtualBox, thankfully). iTunes is slow, clumsy and generally unusable piece of dog excrements. It can only do one thing at a time, mostly: if you are adding a folder to your library, you can t switch to another tab and see how the backup of your iPhone is going. The actions do happen in the background, but more often than not you cann t switch away from some actions. If you try to do multiple things in parallel, instead of resolving any process conflicts and queueing the actions iTunes will simply silently fails to do something, often leaving remains of the half-finished actions. It also does not bother to check if the backup or restore of the phone is complete before using it in this way I have many time gotten into a situation where I connect the iPhone to the iTunes, it starts backing it up, then crashes and next time happily offers this half-done backup copy as a valid restore point. Or it crashes during restore and only writes some files and settings to the phone other settings stay at their default values. It is also very cryptic, for example if you have a lot of music it can take hours to import it into iTunes (it spent 2 hours Analysing gapless playback information of 5000 songs) and then you will be puzzled about how to put some of that on the iPhone. If you just tick the checkbox to sync your music you ll get an error stating that there is not enough free space, which is not very helpful. The magic combination was to check Manually manage music and then go to music tab and choose Autofill option. At that point iTunes will spend half an hour choosing which songs to put on the phone, before it even starts copying. And god forbid that you would use that AppStore application on the iPhone to actually install apps sooner or later you will run into a backup bug where creating or restoring a backup of your iPhone might take 2-3 hours instead of more normal 5-15 minutes. Only a factory reset of the phone followed by selective reinstallation of applications (loosing all their settings in the process) can work around this bug currently.
• The closed nature of the iPhone means that I cann t install unapproved software unless I jailbreak the phone and even then it is rather problematic to develop for the phone unless you have a Mac and shell out 99$ a year for participation in the developers program
• The iPhone is a consumption device and not a productivity device. I would like to have a device on which I could be creative as a software developer as well

Option 2: Maemo/MeeGo Some of my colleagues at work (in the Riga office of Accenture) have N900 phones and I have been exposed to people with raw enthusiasm towards the Maemo platform ever since the Debconf in Helsinki, where we saw the first N700 devices in the hands of some lucky Debian/Nokia people using it as an Internet tablet. N900 has been a strong leap forward for the platform, before its head was teleported sideways by the whole MeeGo merger/debackle. Again I have two options here: Get a N900 either here in Latvia for 325 LVL (that would be the total cost over 2 years) or get it from USA during Debconf for 399 USD (+129 USD for 2 year warranty) = 317 LVL. And additional option would be to wait until October/November and then buy the new Nokia MeeGo device, rumoured to be N9-00 and likely to be around the same price as N900 was when it was introduced (around 500 LVL). Benefits:

• A fully open and rooted Debian Linux based device with apt-get, X, pusleaudio, d-bus and Qt as core technologies
• I can install software from anyone and can also write my own software either in Qt/C++ or even in Python
• There is (or was) a significant hacker community that develops applications for N900
• As far as I know most of the features that I need do work on N900, but I am unsure about:
  1. audiobook support how easy it is to put a MP3 on the device and tell it is an audiobbok so that the device would remember position when playing the file and save it even if the player is stopped and the device is rebooted or backed up and restored from backup?
  2. can I have Google Mail, Contacts and Calendar and at the same time also have an Exchange account with mail and contacts and calendar active?
  3. how is the Geocaching.com support is there an app already that makes it easy to geocache while offline?
  4. Multilingual keyboards I ve seen people having to reimplement the on-screen keyboard in their programs, and naturally they do not bother to add support for all language. So support for multilingual keyboard input is an open one.

Drawbacks:

• All the uncertainties above while I could write all the above software, it would be better if I would not have to, so that I could focus on something more productive
• Also the whole position of Nokia on the MeeGo support on the N900 is kinda backwards. I am now used to the Apple way that if I get a device, then it will get software updates for at least next two years and will get the new software features developed in the next two years at least. If Nokia would have said All phone-oriented MeeGo releases will have a version for N900 until at least the end of 2012. Then this drawback would not have been there, but currently it is a mayor sticking stone for me. I don t want to buy a device that will be on a dead-end software platform that will die before the end of this year.
• N900 is the largest and heaviest of all the options

Option 3 Android After looking trough the options for Android phones, I ve narrowed the selection to the ones that are either available from local carriers or can be easily gotten unlocked also the phones need to have announced plans to have at least Android 2.2 version. Currently the choice is limited to HTC Desire (from carrier or unlocked) and Samsung Galaxy S (unlocked). The prices over 2 years break down as follows: HTC Desire (locked to LMT): 69 + 9*24 = 285 LVL
HTC Desire (locked to Bite): 199 + 2*24 = 247 LVL (Note: only 500Mb of data per month available)
HTC Desire (locked to Tele2): 179 + 6*24 = 323 LVL
Note: all the above options would also require me to pay a 45 LVL early termination fee on my current iPhone contract if I choose to do this before September.
HTC Desire (unlocked): 311 LVL
Samsung Galaxy S (unlocked): 350 LVL (import from Germany) Benefits:

• As far as I could find, the features I need are out there in one way or another there usually is an app for that!
• Google looks poised to continue development at a rapid pace
• There is an active hacker community and also an active commercial software community that offers cool applications for a few bucks
• As far as I understand the phone backups onto itself a backup of the phone is created onto a SD card in the phone and you can then copy that off the phone for permanent backup, it is a nice concept
• There are literally thousands of sources for Android applications: from the official Market to Google Code to individual web pages and forums
• The hardware looks solid, powerful and generic enough to last for a couple of years
• It is likely that knowing how to program Android app will be useful for me at work
• It is cheap and simple to start writing Android apps on Linux

Drawbacks:

• Typical usability of Android apps is pretty low compared to iPhone
• The Android environment is very unique and is not like anything else it s not really much of a Linux system from an apps perspective
• I would be relying on HTC to provide OS updates in a timely manner it is quite likely that after a year or so the updates to new version will come slowly or even stop altogether and I will have to use hacked ROMs
• I have a dislike for Java. I wish one could write fully featured Android apps in Python

Please correct me if I am wrong with something!!! And also it would be cool if you expressed your own opinions in the comments. Currently I am very undecided about what I am going to do, but after completing this entry I am leaning towards an unlocked HTC Desire. Update:
Olivier Cr te shares a N900 vs. Nexus One experience Edit 01-07-2010: Handed my iPhone in for a warranty repair, got a dumbphone Nokia as a loaner during repairs. Very surprised about how long a battery can last on a low powered and cheap phone. In the mean time from all the comments here and elsewhere I am starting to see that I am too annoyed with the iPhone platform (mostly iTunes) to stay there and that I am also not convinced in the direction MeeGo is going (a lot of community developers are annoyed and are jumping ship), so N900 is also out. I will need to see 2-3 MeeGo smartphones and also see how Nokia will treat their smartphone users and developers on this platform, in the long term, before I ll be ready to trust them with my money. Therefore, my choice is becoming pretty clear look for the best Android phone (after my iPhone dies). We have this thing at Accenture, where we can get company phones at a discount and HTC Desire is on that list. If I can get that, it would slash 100 LVL (almost a third of the price!) off it, but to get there I either need to get a promotion (rather rare after just one year with the company) or prove the necessity of the phone for the needs of my project (kinda hard currently). Well, the situation will be a bit clearer next week when I get my iPhone back. (Apparently the Latvia s largest holiday Midsummer Festivities or J i/L go with tons of traditional outdoor activities, caused a lot of phones to become broken and there is a backlog of warranty service ) So at this point my plan is: get my iPhone repaired and get a promotion or a project where I can justify a company-paid HTC Desire and then hope that Nokia/Intel really get their stuff together and make MeeGo a great smartphone platform over the next year or two. Update: Got my iPhone 3G back from the warranty . It is in quotes because they just gave me a new iPhone 3G. The battery lasts for 3 days of minimal use and it has not crashed yet. The recovery process was a pain again caused by iTunes: iTunes refused to restore my backup onto the new phone, because my backup was made on a iOS 4.0, while the new phone I got from the warranty has 3.1.3 on it. So I had to set this phone as a new phone, initialise it, make a backup, upgrade to 4.0, restore from the (empty) backup, then reset the phone to factory settings again and only then I could start restoring data from my original (pre-warranty) backup and start copying my data on to the device. All in all it took nearly 3 hours if iTunes doing something and almost a dozen reboots of the phone. Almost half of that time was spent doing completely unnecessary steps to work around the fact that iTunes is braindead. I ll continue using my iPhone for now, keeping my eyes on the newest Android phones and also waiting for the long-promised first MeeGo phone from Nokia.

check-deps.sh
Phil Hands and I devised a little script which is currently packaged as part of multistrap to check the dependencies of a .deb package before installing it - called check-deps.sh and can be found at /usr/share/multistrap/check-deps.sh. I just thought I'd publicise it a bit because it can help with one of my common test cases - when a binary package changes dependencies but you haven't uploaded it yet.

check-deps.sh takes a .deb to the command line -f --file option and outputs the dependencies of that .deb and whether those dependencies (at the requested versions) exist in your apt-cache policy. Optionally, you can use check-deps.sh to also then install those dependencies (not the dependencies that would come from the current version in the repository) and the package itself.

I've found this to be a much more friendly way of satisfying runtime dependencies of a package. Where things like pbuilder would just force install the .deb and then get apt-get -f install to try and fix things, check-deps.sh will fail if the required dependencies are not available in the first place.

The benefits are two-fold. First you get to see the full list and you can easily see if your work to drop a dependency has just resulted in another package bringing that dependency back in etc. Secondly, you can test whether your .deb is installable without breaking your system and then having to remove it - e.g. against backports or a Debian derivative test chroot. It should be possible to extend the script to read Build-Depends but that means reading the input from the source package, not the built binary. Might be just as easy to have a second script for Build-Depends.

(Comments are still disabled due to spammers and time-wasters, but if you would like to see check-deps.sh more widely used or extended it isn't that hard to find my email address at debian.org.)

xapt
Another new perl script, this time using the multistrap and apt-grip core to replace the apt-cross breakage to install cross-dependencies of a Debian source package. xapt has very few dependencies (dpkg-dev, dpkg-cross, apt, perl) and very simple logic. It doesn't try to second-guess which packages to use as dependencies of cross packages, it just gets them all. As such, it still isn't the tool we need for optimal cross-dependency resolution, but then neither is apt-cross - come to that, Multiarch or sysroot aren't going to provide the right tool any time soon either. So xapt at least gets over the worst of the design problems in apt-cross (by not using the apt perl bindings mainly) and seems to just work within clean chroots which is where apt-cross has the most problems.

xapt will be a new binary package, so it will need to go through NEW but I'm looking at using it as part of pdebuild-cross because it is faster than apt-cross and seems to work much more efficiently than apt-cross inside a clean chroot. xapt does not care about multiarch or sysroot, it just gets the packages and passes them to dpkg-cross. Normally, xapt cleans up after itself but you can choose to leave the built packages in /var/lib/xapt/output if that is useful.

Note to self: It is actually possible to select a single test module or a single test function in py.test. But passing the file name as argument to a py.test invocation selects only the doctest from that file (wtf!?). Instead, you need to call it like this: This is documented in the section advanced test selection and running modes of the py.test documentation, although I fail to see how this is advanced.

My project. This post assumes that the reader has read my last post. I m posting this weekly report earlier this week because there are too many things to tell already. I ve found the reason behind the segfault I m looking for so much time. The last problem, which is the only I know exactly when was fixed, because the programs started working, was related to allocating two times in the same region of memory. This happened because in each major GC the list of free line groups is generated again, but my old code was still allocating in the same line group of the last generation. So the last part of the line group, which was not yet used, would be a part of a line group in generated in the new collection, and it will be used for allocation two times: one in the allocation of the current line group, and another when this new line group starts being allocated. The implementation of the allocation of memory in lines is not very complicated, but it has some details that should be paid attention, and that were the cause of most trouble last weeks, and still need improvement. Initially I was allocating one object per line, just to see if it would work. As it didn t, I kept on improving the approach until I could find the problem. The next attempt was by setting ws->todo_free and ws->todo_lim in alloc_for_copy() in rts/sm/Evac.c. I think this is not ideal, because I didn t want the code to become too inconsistent with the way memory was allocated using these pointers before my changes. So I created new variables, line_free and line_lim, at first in the gen_workspace ws, the same place that todo_free and todo_lim are, but because of the last problem I described in the previous paragraph I changed it to generation. I m still not sure about where to place these pointers, this is something that can be improved. Another problem that I took a long time to understand is that the object need to be scavenged after being allocated. When it was allocated in todo_free, it was being scavenged by scavenge_block() in rts/sm/Scav.c, because the block in which it s in, todo_bd, is scavenged by this function. As I didn t wanted to the whole block where the free line group is to get scavenged again, I didn t want to send it to this function. So I thought about creating a way to scavenge only part of a block, that is, the space in the free line group that was allocated. This is still a valid idea, but I noticed that it was easier to use the mark stack. So I mark the object that is allocated in the line and push it in the mark stack. The main problem with this approach it s only possible to allocate in lines during major GCs, since only in this kind of GCs the mark stack is active. This is certainly the place where I can make more improvement. The patch of these changes and another one for the sanity checking explained in the last post. I m now benchmarking these changes with nofib, to see how much it affects the performance.

I have a lot of papers lying around. I now try to keep them organized, but I haven't got through the back log of when I didn't organize them. Sometimes I come across stuff that I'm happy I had in the pile. The following was my attempt at a valedictorian speech for the 2005 Math class at the University of Waterloo. I was not the valedictorian, but I tried, and thus have a speech.

Ladies, Gentlemen, Doctors and children of all ages. Thank you for spending our last day at Waterloo, here with us. I am one of those people who won't be staying here after day, but somehow I was still chosen to be the valedictorian. So, when I was asked to be the valedictorian by some friends of mine, I was thrilled. I also though I could do a good job, heck, I got my degree in CS, therefore, I can do anything. So, I accepted the nomination. I looked at the nomination form and didn't find any hints on what to talk about. At this point I realized that this speech may be a little underspecified. In fact, there was no specification at all.

I gathered what I could from the form, i.e. a speech had to be written by, and performed on the 18th of June 2005. Skip forward a few weeks, I'd accomplished nothing more than vague thoughts on what to talk about, then I got an email. All potential speeches had to be presented on the 6th of April 2005 (the deadline had moved up, and still no specification). However, a performance objective was given in that email -- each speech must be completed in 5 minutes. Given that I don't have a spec, I can only hope no one else has one.

What follows is my implementation of this unknown specification.

Another cycle in our lives is over. We are finished our latest round of education. This time from the University of Waterloo, in some Math program. Many of us have changed significantly while we've been here, but we are still the same people. We know we are older, we can guess that we are wiser, and we are certainly more educated. So, here is a quick tour of my cycle through Waterloo.

When I got here back in September 2000, I was amazed at how many smart people could be put together in one small place. I am actually still amazed by all the smart people here. I simply happen to know a bit more about some of these people and the amazing things they have done. With all these smart people here, I even had to do homework to keep up.

Skip forward to second year. Second year is when the university decided there is no more room for me in residence. In fact, they do that to almost all second year students. So, I found a sleezy place to live off-campus and started discovering what life is like outside the university bubble. For example, there are stores not in the plaza. There are a couple malls, a bus station, and a train station.

By third year Waterloo had become home despite working in two other cities and growing up on the other side of Toronto. I no longer had to fear being kicked out of CS since I had enough courses with good marks racked up that my cumulative average wouldn't drop too low unless I started failing, which would have been a bad thing to do.

In fourth year, I could see the end in sight. Grad school applications needed to be done pronto and some interesting job prospects lured me away from yet more years of school. Co-op didn't help keep me in school, I enjoyed the working world. I know there are others that used co-op to find that the working world wasn't for them. I expect those people to be in some school for a good long time.

Now we are at the end, and the cycle is complete. I've been across the country, lived over in sunny California, and biked through Scotland, but no where has compared to Waterloo, or the great people that make it such a wonderful place to be. I'm sad to go, as I'm sure many of you are, but I hope you are excited about the next step at the same time.

That was my 5 years, in a non-chronologically accurate order. But it's been even better to help and see people do amazing things. Small things, like getting a group of friends together to go get pints of Bubbletea. To large things like running frosh week for thousands of first year students.

Now that we have looked at the past, what about the future? One of the things I didn't wan to say during this speech is that "We are the future". What does that even mean? Are we useless now? I don't know what it means, it could even be partially true, but we are not useless. In fact our time is now! Among this graduating class are those who are doing cutting edge research. There are those we are already running their own companies. There are those who are already half-way through there plan to rule the world. There are even those of us who are simply going to help others do those things. So, we are not "the future", we are "the now".

Being "the now", (don't I sound like Dubya?) is one thing, but we should always be "the now". We can always make now be our time. We should do this because if we aren't the ones living happily and making the world a better place, in our own ways, then who is going to do it?

If nothing else, I really hope everyone lives happily. For me, I found if I can keep my hat then I can be happy. Well, I can be happy promoting "No Pants Day"!

DOWN WITH PANTS!!

At the end, I finally figured out the specification for this speech. It's simple: Remind people of the awesome things they have done here, the amazing people they shouldn't forget. Lastly, to convince people to keep doing the awesome things despite what might happen or what obstacles appear.

Thank you, I enjoyed being your VD today.

Today, virtually all of the free software available can be found in packaged form in distributions like Debian and Ubuntu. Users of these distributions have access to a library of thousands of applications, ranging from trivial to highly sophisticated software systems. Developers can find a vast array of programming languages, tools and libraries for constructing new applications. This is possible because we have a mature system for turning free software components into standardized modules (packages). Some software is more difficult to package and maintain, and I m occasionally surprised to find something very useful which isn t packaged yet, but in general, the software I want is packaged and ready before I realize I need it. Even the long tail of niche software is generally packaged very effectively. Thanks to coherent standards, sophisticated management tools, and the principles of software freedom, these packages can be mixed and matched to create complete software stacks for a wide range of devices, from netbooks to supercomputing clusters. These stacks are tightly integrated, and can be tested, released, maintained and upgraded as a unit. The Debian system is unparalleled for this purpose, which is why Ubuntu is based on it. The vision, for a free software operating system which is highly modular and customizable, has been achieved. Rough edges This is a momentous achievement, and the Debian packaging system fulfills its intended purpose very well. However, there are a number of areas where it introduces friction, because the package model doesn t quite fit some new problems. Most of these are becoming more common over time as technology evolves and changes shape.

• Embedded systems need to be pared down to the essentials to minimize storage, distribution, computation and maintenance costs. Standardized packaging introduces excessive code, data and interdependency which make the system larger than necessary. Tight integration makes it difficult to bootstrap the system from scratch for custom hardware. Projects like Embedded Debian aim to adapt the Debian system to be more suitable for use in these environments, to varying degrees of success. Meanwhile, smart phones will soon become the most common type of computer globally.
• Data, in contrast to software, has simple requirements. It just needs to be up to date and accessible to programs. Packaging and distributing it through the standardized packaging process is awkward, doesn t offer tangible benefits, and introduces overhead. There have been extensive debates in Debian about how to handle large data sets. Meanwhile, this problem is becoming increasingly important as data science catalyzes a new wave of applications.
• Client/server and other types of distributed applications are notoriously tricky to package. The packaging system works within the context of a single OS instance, and so relationships which span multiple OS instances (e.g. a server application which depends on a database running on another server) are not straightforward. Meanwhile, the web has become a first-class application development platform, and this kind of interdependency is extremely common on both clients and servers.
• Cross-platform applications such as Firefox, Chromium and OpenOffice.org have long struggled with packaging. In order to be portable, they tend to bundle the components they depend on, such as libraries. Packagers strive for normalization, and want these applications to use the packaged versions of these libraries instead. Application developers build, test and ship one set of dependencies, but their users receive a different stack when they use the packaged version of the application. Developers on both sides are in constant tension as they expect their configuration to be the canonical one, and want it to be tightly integrated. Cross-platform application developers want to provide their own, application-specific cross-platform update mechanism, while distributions want to use the same mechanism for all their components.
• Virtual appliances aim to combine application and operating system into a portable bundle. While a modular OS is definitely called for, appliances face some of the same problems as embedded systems as they need to be minimized. Furthermore, the appliance becomes a component in itself, and requires metadata, distribution mechanisms and so on. If someone wants to install a virtual appliance, how should that work? Packaging them up as .debs doesn t make much sense for the same reasons that apply to large data sets. I haven t seen virtual appliances really taking off, but I expect cloud to change that.
• Runtime libraries for languages such as Perl, Python and Ruby provide their own packaging systems, which manage dependencies and other metadata, installation, upgrades and removal in a standardized way. Because these operate independently of the OS package manager, all sorts of problems arise. Projects such as GoboLinux have attempted to tie them together, to varying degrees of success. Meanwhile, each new programming language we invent comes with a different, incompatible package manager, and distribution developers need to spend time repackaging them into their preferred format.

Why are we stuck?

I suppose it is tempting, if the only tool you have is a hammer, to treat everything as if it were a nail.
Abraham Maslow

The packaging ecosystem is very strong. Not only do we have powerful tools for working with packages, we also benefit from packages being a well-understood concept, and having established processes for developing, exchanging and talking about them. Once something is packaged, we know what it is and how to work with it, and it fits into everything else. So, it is tempting to package everything in sight, as we already know how to make sense of packages. However, this may not always be the right tool for the job. Various attempts have been made to extend the packaging concept to make it more general, for example:

• Portage, of Gentoo fame, offers impressive flexibility by building packages with a custom configuration, tailored for the needs of the target system.
• Conary, from rPath, offers finer-grained dependencies, powerful revision control and object-oriented build recipes.
• Nix provides a consistent build and runtime environment, ensuring that programs are run with the same dependencies used to build them, by keeping the relevant versions installed. I don t know much about it, but it sounds like all dependencies implicitly refer to an exact version.

Other package managers aim to solve a specific problem, such as providing lightweight package management for embedded systems, or lazy dependency installation, or fixing the filesystem hierarchy. There is a long list of package managers of various levels which solve different problems. Most of these systems suffer from an important fundamental tradeoff: they are designed to manage the entire system, from the kernel through applications, and so they must be used wholesale in order to reap their full benefit. In other words, in their world, everything is a package, and anything which is not a package is out of scope. Therefore, each of these systems requires a separate collection of packages, and each time we invent a new one, its adherents set about packaging everything in the new format. It takes a very long time to do this, and most of them lose momentum before a mature ecosystem can form around them. This lock-in effect makes it difficult for new packaging technologies to succeed. Divide and Conquer No single package management framework is flexible enough to accommodate all of the needs we have today. Even more importantly, a generic solution won t account for the needs we will have tomorrow. I propose that in order to move forward, we must make it possible to solve packaging problems separately, rather than attempting to solve them all within a single system.

• Decouple applications from the platform. Debian packaging is an excellent solution for managing the network of highly interdependent components which make up the core of a modern Linux distribution. It falls short, however, for managing the needs of modern applications: fast-moving, cross-platform and client/server (especially web). Let s stop trying to fit these square pegs into round holes, and adopt a different solution for this space, preferably one which is comprehensible and useful to application developers so that they can do most of the work.
• Treat data as a service. It s no longer useful to package up documentation in order to provide local copies of it on every Linux system. The web is a much, much richer and more effective solution to that problem. The same principle is increasingly applicable to structured data. From documents and contacts to anti-virus signatures and PCI IDs, there s much better data to be had out there on the web than down here on the local filesystem.
• Simplify integration between packaging systems in order to enable a heterogeneous model. When we break the assumption that everything is a package, we will need new tools to manage the interfaces between different types of components. Applications will need to introspect their dependency chain, and system management tools will need to be able to interrogate applications. We ll need thoughtfully designed interfaces which provide an appropriate level of abstraction while offering sufficient flexibility to solve many different packaging problems. There is unarguably a cost to this heterogeneity, but I believe it would easily outweigh the shortcomings of our current model.

But I like things how they are! We don t have a choice. The world is changing around us, and distributions need to evolve with it. If we don t adapt, we will eventually give way to systems which do solve these problems. Take, for example, modern web browsers like Firefox and Chromium. Arguably the most vital application for users, the browser is coming under increasing pressure to keep up with the breakneck pace of innovation on the web. The next wave of real-time collaboration and multimedia applications relies on the rapid development of new capabilities in web browsers. Browser makers are responding by accelerating deployment in the field: both aggressively push new releases to their users. A report from Google found that Chrome upgrades 97% of their users within 21 days of a new release, and Firefox 85% (both impressive numbers). Mozilla recently changed their maintenance policies, discontinuing maintenance of stable releases and forcing Ubuntu to ship new upstream releases to users. These applications are just the leading edge of the curve, and the pressure will only increase. Equally powerful trends are pressing server applications, embedded systems, and data to adapt as well. The ideas I ve presented here are only one possible way forward, and I m sure there are more and better ideas brewing in distribution communities. I m sure that I m not the only one thinking about these problems. Whatever it looks like in the end, I have no doubt that change is ahead.

My girlfriend and I were out picking up a cake at the bakery last week, when my phone rang:

• Hello, I'm calling about your application. I just spent the last hour discussing your case with my boss and there's just one thing that bothers us: do you ever intend on paying your residual taxes?
• I received those payment slips right after I became unemployed and, as you know, living off unemployment benefits in this country's most expensive city doesn't exactly leave anyone with money to spare...
• Say no more! Moving to the capital for this job was a real shocker. The rents are so bloody expensive here! Anyhow, do you intend on taking care of those residual taxes as soon as you get a job?
• Yup, just like I said in my application.
• Alright, then I guess that everything is in order. We can start processing your application today. We obviously cannot make any promise about how long that's gonna take, but I would think that the decision should come fairly soon.
• Wow! That's excellent news! Thanks again for your help!
• You're welcome, sir. Have a nice day!

Without knowing, at that moment, I had just become a citizen of the country in which I have been living for the past 12 years. It was only yesterday, upon receiving the decision in the mail and looking at the date on the certificate that I realized that, when I got the phone call, the decision had already been made and the bureaucracy was only looking for reassurances that I fully intend upon acquitting my civic obligations as soon as humanely possible. To say that reading the decision was a highly emotional moment is an understatement. Trying to explain the intensity of this moment to my girlfriend, I compared it to the day when a teenager reaches adulthood. This instant brought a similar feeling: suddenly, the whole EU opens its doors to me and I'm free to decide how to best use the opportunities it offers. What next? For now, completing this government training to become a bureaucrat. Funnily enough, becoming a citizen resolved the whole issue of background checks, which also suddenly triples the number of possible venues for the on-job part of the training. In my case, it looks like I'll be spending the next few months at the Ministry of Employment and Entrepreneurship, working on EU projects that fund R&D efforts and export sales ventures in each member state. After that, I'm not sure. On one hand, I'd like to apply for the Ministry of Foreign Affairs' KAVAKU recruitment program for future diplomats. On the other hand, that ministry is extremely picky about whom it accepts and it's not particularly known for favoring naturalized individuals. This being said, our current Minister recently published rather ambitious plans to completely reform the Ministry by bringing in seasoned professionals from the private sector who could efficiently promote Finnish know-how and products abroad, rather than hiring more of the same Public Administration graduates, so, who knows? Maybe the time is right for someone like me to join the ranks of the Finnish diplomacy? An other option that I'm considering is to permanently move to Estonia. When my last job there ended, I was left with the feeling that I could have accomplished a lot more, if only I were in a legal position to move there, rather than commute a couple of times a week. Beyond the pioneering work that myself and my diplomat friends at the Estonian embassy did in Turkey, there was a demand for us to perform the same magic in other countries of interest to Estonia. Unfortunately, not being in a position to be on-site and no longer having a job that paid for me to be there often enough meant that I had to pass on that opportunity. Now, seeing how one of my friends recently left the diplomacy and is open to new challenges, I'm wondering if now might be a good time to resume our operation and prepare our next campaign? Wow... So much to think about, now that a whole continent opened its doors to me. Hienoa! Olen suomalainen.

The current version of mksh had use of arc4random(3) removed, including set -o arc4random , to speed it up (on some architectures, a lot) this will break some existing scripts (such as /etc/rc *cough* on MirBSD ). Hence I decided to publish the next version of mksh(1) as R40 based upon current development, and defer plans for associative arrays (and multidimensional arrays) for mksh R41. There s also already the change to Build.sh arguments, so this suits me quite fine. (Read: if running MirBSD, don t upgrade mksh at the moment.) There will be a new MirBSD snapshot once this is fixed, maybe a few more changes to the shell for better POSuX compliance, and the recently mentioned patent on LFNs (long filename) in FAT will be taken into account with a patch to msdosfs. I ll travel to LSM/RMLL 2010, the Libre Software Meeting (Rencontres Mondiales du Logiciel Libre) tomorrow until the weekend, to hack some on FusionForge (this is worktime for me), visit XTaran, Uriel, and maybe a couple of other usual people . Thundersday, between 10:00 UTC and 12:00 UTC, eurynome will be shut down by gecko2@ due to power supply maintenance on the host system data centre. We have a new mirror in the Americas, thanks a lot to Mike 'Fuzzy' Partin! Benny will mention it on the webpages once it s working.

Wednesday, after some weeks with a flickering screen (or more precisely, something is causing GTK+ to redraw and the kernel to print ^@ in the terminal when you touch the screen, see http://jak-linux.org/tmp/20100628_002.mp4) in my 3 years old HP Compaq 6720s (which seems to be a software-hardware combination problem, at least it works in Ubuntu); I decided to buy a new laptop. It took me a few minutes to find the Lenovo ThinkPad Edge 15 NVL7VGE; which I ordered at notebooksbilliger.de at around 14:30 CEST using Express. On Thursday morning at 6:40 CEST, the device arrived. The laptop features an Intel Core i5 430 M processor with integrated graphics, 4 GiB of RAM, 320 GB hard disk storage, matte screen, and of course a TrackPoint. The machine comes pre-installed with Windows 7 Professional which is not very nice (and Lenovo does not take Windows licenses back). I booted Ubuntu 10.04 from an USB disk which worked a few times, but not always, so I upgraded the BIOS to make USB booting reliable (as Lenovo published multiple BIOS updates already, one fixing this USB booting issue). The first step was booting Ubuntu 10.04 from an USB hard disk to check the Linux support. Using Ubuntu, everything worked out of the box, including stuff like HDMI audio support or output switching. An exception may be the DVD drive, which only works if you set the SATA mode to compatibility. Non-DVD media also works in AHCI mode, but only if you start with a disc inserted in the drive. Playing DVDs requires setting a region code using setregion(8) [otherwise they do not work at all] and SATA compatibility mode [otherwise they only work partially]. Another issue is the HDMI output, as the TV is not correctly detected and some parts at the top and the bottom of the image are missing (that is, the GNOME panels) (but that happens with different screens and ATI cards as well). Oh, and closing the lid with a screen attached and the internal screen disabled works only as long as you do not open it again, because that causes the GPU to stop. On the input side, we have one keyboard, one TrackPoint, and a touch pad. All of them work out of the box, but they have a few problems on the hardware side: The cursor keys on the keyboard are incredibly small and the touchpad is very large. The keyboard is different from the one in the HP Compaq: First of all, there is space between the keys (like Apple keyboards); and secondly, you need to apply more pressure in order to use the keys. The device feels more solid than the HP Compaq 6720s, especially touching the screen does not change the colors on the screen. I don t know how long the keyboard will last, but I hope that I will still have symbols on them in 5 months (which was not the case for the HP Compaq 6720s keyboard). Power consumption is another interesting point. When the device is idle and the screen is disabled, the device seems to consume about 10W. With screen enabled, consumption varies between 13W (idle) 35W (building source code); if the values received via ACPI are correct. The device ships with a 48 Wh battery which after some time was displayed as 53 Wh in gnome-power-statistics (only the when loaded state, capacity is displayed as 47.5Wh). According to Lenovo, the laptop lasts 4.5 hours on battery, meaning consumption of 47.5 Wh / 4.5 h = 10.5 Wh which is a bit overly optimistic if you want to work on the device. After checking everything, I took over my Debian sid installation from my old laptop (which I installed in 2008). It runs happely now, although I still need to get accustomed to the new keyboard and the large touch pad. All in all, the price of 765 I paid for the device seems to be OK. Now I only need to remove the Windows 7 sticker and hope that the device lasts long enough until I want to buy a new one (probably 2013, if the aliens don t invade us 2012).
Filed under: General

to start with the good news: christine has joined the RCBW initiative yeah! let's see who joins next ...

as usual, a short overview about my own attempts in the RC bugs department:

• #580106 libclutter-1.0-0: "empathy: fails to start on armel"
  grab patch from BTS and upload to DELAYED/2
• #583465 src:clutter-1.0: "clutter-1.0: FTBFS: ERROR in annotation: can't find parameter n_properies referenced by parameter values of 'animatev'"
  test, add a comment to the BTS, grab patch from BTS and upload to DELAYED/2
• #583648 zoneminder: "package removed, processes still running"
  apply patch for debian/postinst from BTS and upload to DELAYED/2
• #585410 nagios3-cgi: "nagios3-cgi: Missing conflict with nagios3 v3.0.6-4~lenny2 (/usr/lib/cgi-bin/nagios3/extinfo.cgi_"
  add Replaces/Breaks and upload to DELAYED/2, later cancelled on maintainer's request; later fixed by a maintainer upload based on my patch
• #587830 libnet-sftp-foreign-perl: "libnet-sftp-foreign-perl: causes script to fail silently with exit code 0"
  add missing dependencies (pkg-perl)

I hate keeping track of money. Bank accounts, credit cards, investing it's such a hassle. Hell, sometimes I even hate the fact that money exists and needs to be dealt with in the first place. But I admit that it's an easier system than bartering for everything. In college I pretty much ignored most things financial. I rarely had more money than I needed, and put little financial planning into deciding what I could and could not spend money on besides, "I'm going to make X dollars this summer," and, "think twice." But now that I have a steady paycheck, it seems like a good idea to know where all my money goes, so I can make better decisions about how I'm spending (and saving) it. But I've kept procrastinating starting to do so, because GUI programs like Gnucash and HomeBank seem like such a hassle. Their first screen is dauntingly complex, and if you don't know much about accounting it's scary and difficult to be asked to set up a big set of accounts when first starting to use the program, without having any prior experience with what you personally would find useful to keep track of. Entering data through menus and dialogs is tedious and slow. hledger (or ledger, which came first) had been appearing on my radar recently, not least because Iron Blogger uses it). hledger changed my opinion of accounting software. In about 15 minutes, using only the sample transactions from the manual, I was able to enter all my assets and liabilities bank accounts, credit card, student loans, money I've borrowed from people and never paid back though I said I would, etc. And then, I could run 'hledger balance' and it would tell me what sorts of things I had spent money on in the past few days, as far back as I could bother looking up actual transactions for, rather than entering a single transaction with a balance forward. Duuude, awesome! I'm a geek. I like statistics and data about my life, as long as it's not a huge pain to collect the data in the first place. More data means more on which to base decisions, decisions which will then be freer from the bias of what I find memorable enough to remember having done. Here's what I like best about ledger:

• Command-line interface for the hacker in me.
• Pre-existing facility with a text editor transfers over to facility of data entry.
• No need to set up accounts separately from transactions. Transactions are the important thing, and accounts just automagically appear when the account name appears in a transaction. Mistakes are trivial to correct in a text editor.
• Easy to start out with "use text editor to add transactions" and "hledger balance" and then branch out to more advanced features as necessary.
• Text file format is well-suited to storing in a version control system.
• It easily replaces two text files that I used to keep: "money owed to others" and "checks written but not yet cashed by the other party."

I don't think I'd be enjoying keeping track of my money nearly so much without a tool like hledger. It gets out of your way to let you focus on the hard things, like choosing categories for the things you spend money on and remembering to record the data in the first place.

Here is a short update on my my Debian Lenny->Squeeze upgrade testing. Here is a summary of the difference for Gnome when it is upgraded by apt-get and aptitude. I'm not reporting the status for KDE, because the upgrade crashes when aptitude try because of missing conflicts (#584861 and #585716). At the end of the upgrade test script, dpkg -l is executed to get a complete list of the installed packages. Based on this I see these differences when I did a test run today. As usual, I do not really know what the correct set of packages would be, but thought it best to publish the difference. Installed using apt-get, missing with aptitude

at-spi cpp-4.3 finger gnome-spell gstreamer0.10-gnomevfs libatspi1.0-0 libcupsys2 libeel2-data libgail-common libgdl-1-common libgnomeprint2.2-data libgnomeprintui2.2-common libgnomevfs2-bin libgtksourceview-common libpt-1.10.10-plugins-alsa libpt-1.10.10-plugins-v4l libservlet2.4-java libxalan2-java libxerces2-java openoffice.org-writer2latex openssl-blacklist p7zip python-4suite-xml python-eggtrayicon python-gtkhtml2 python-gtkmozembed svgalibg1 xserver-xephyr zip

Installed using apt-get, removed with aptitude

bluez-utils dhcdbd djvulibre-desktop epiphany-gecko gnome-app-install gnome-mount gnome-vfs-obexftp gnome-volume-manager libao2 libavahi-compat-libdnssd1 libavahi-core5 libbind9-50 libbluetooth2 libcamel1.2-11 libcdio7 libcucul0 libcurl3 libdirectfb-1.0-0 libdvdread3 libedata-cal1.2-6 libedataserver1.2-9 libeel2-2.20 libepc-1.0-1 libepc-ui-1.0-1 libexchange-storage1.2-3 libfaad0 libgd2-noxpm libgda3-3 libgda3-common libggz2 libggzcore9 libggzmod4 libgksu1.2-0 libgksuui1.0-1 libgmyth0 libgnome-desktop-2 libgnome-pilot2 libgnomecups1.0-1 libgnomeprint2.2-0 libgnomeprintui2.2-0 libgpod3 libgraphviz4 libgtkhtml2-0 libgtksourceview1.0-0 libgucharmap6 libhesiod0 libicu38 libisccc50 libisccfg50 libiw29 libkpathsea4 libltdl3 liblwres50 libmagick++10 libmagick10 libmalaga7 libmtp7 libmysqlclient15off libnautilus-burn4 libneon27 libnm-glib0 libnm-util0 libopal-2.2 libosp5 libparted1.8-10 libpisock9 libpisync1 libpoppler-glib3 libpoppler3 libpt-1.10.10 libraw1394-8 libsensors3 libsmbios2 libsoup2.2-8 libssh2-1 libsuitesparse-3.1.0 libswfdec-0.6-90 libtalloc1 libtotem-plparser10 libtrackerclient0 libvoikko1 libxalan2-java-gcj libxerces2-java-gcj libxklavier12 libxtrap6 libxxf86misc1 libzephyr3 mysql-common swfdec-gnome totem-gstreamer wodim

Installed using aptitude, missing with apt-get

gnome gnome-desktop-environment hamster-applet python-gnomeapplet python-gnomekeyring python-wnck rhythmbox-plugins xorg xserver-xorg-input-all xserver-xorg-input-evdev xserver-xorg-input-kbd xserver-xorg-input-mouse xserver-xorg-input-synaptics xserver-xorg-video-all xserver-xorg-video-apm xserver-xorg-video-ark xserver-xorg-video-ati xserver-xorg-video-chips xserver-xorg-video-cirrus xserver-xorg-video-dummy xserver-xorg-video-fbdev xserver-xorg-video-glint xserver-xorg-video-i128 xserver-xorg-video-i740 xserver-xorg-video-mach64 xserver-xorg-video-mga xserver-xorg-video-neomagic xserver-xorg-video-nouveau xserver-xorg-video-nv xserver-xorg-video-r128 xserver-xorg-video-radeon xserver-xorg-video-radeonhd xserver-xorg-video-rendition xserver-xorg-video-s3 xserver-xorg-video-s3virge xserver-xorg-video-savage xserver-xorg-video-siliconmotion xserver-xorg-video-sis xserver-xorg-video-sisusb xserver-xorg-video-tdfx xserver-xorg-video-tga xserver-xorg-video-trident xserver-xorg-video-tseng xserver-xorg-video-vesa xserver-xorg-video-vmware xserver-xorg-video-voodoo

Installed using aptitude, removed with apt-get

deskbar-applet xserver-xorg xserver-xorg-core xserver-xorg-input-wacom xserver-xorg-video-intel xserver-xorg-video-openchrome

I was told on IRC that the xorg-xserver package was changed in git today to try to get apt-get to not remove xorg completely. No idea when it hits Squeeze, but when it does I hope it will reduce the difference somewhat.

Starting with GDB version 7, first appearing in Ubuntu with Karmic (Ubuntu 9.10), ASLR is turned off (via the ADDR_NO_RANDOMIZE personality flag) for the debugged process. If you want a more realistic view of how a process will appear in memory, you must set disable-randomization off in gdb:

$ gdb /usr/bin/something
...
(gdb) show disable-randomization
Disabling randomization of debuggee's virtual address space is on.
(gdb) start
...
(gdb) ^Z
$ cat /proc/$(pidof /usr/bin/something)/personality
00040000
$ grep 0040000 /usr/include/linux/personality.h
    ADDR_NO_RANDOMIZE =     0x0040000,  /* disable randomization of VA space */
$ fg
(gdb) set disable-randomization off
(gdb) show disable-randomization
Disabling randomization of debuggee's virtual address space is off.
(gdb) start
...
(gdb) ^Z
$ cat /proc/$(pidof /usr/bin/something)/personality
00000000