diff -urN mogilefs-server-2.30.orig/lib/MogileFS/Store/Drizzle.pm mogilefs-server-2.30/lib/MogileFS/Store/Drizzle.pm --- mogilefs-server-2.30.orig/lib/MogileFS/Store/Drizzle.pm 1969-12-31 16:00:00.000000000 -0800 +++ mogilefs-server-2.30/lib/MogileFS/Store/Drizzle.pm 2009-03-09 10:44:57.000000000 -0700 @@ -0,0 +1,496 @@ +package MogileFS::Store::Drizzle; +use strict; +use warnings; +use DBI 1.44; +use DBD::drizzle; +use MogileFS::Util qw(throw); +use base 'MogileFS::Store'; + +# +# Copyright Clint Byrum , 2009 +# Licensed under the same dual license terms as mogilefsd, Artistic or GPLv2 +# + +# -------------------------------------------------------------------------- +# Package methods we override +# -------------------------------------------------------------------------- + +sub dsn_of_dbhost { + my ($class, $dbname, $host, $port) = @_; + return "DBI:drizzle:$dbname;host=$host" . ($port ? ";port=$port" : ""); +} + +sub dsn_of_root { + my ($class, $dbname, $host, $port) = @_; + return $class->dsn_of_dbhost('drizzle', $host, $port); +} + +# -------------------------------------------------------------------------- +# Store-related things we override +# -------------------------------------------------------------------------- + +sub init { + my $self = shift; + $self->SUPER::init; + $self->{lock_depth} = 0; + $self->{slave_next_check} = 0; +} + +sub post_dbi_connect { + my $self = shift; + $self->SUPER::post_dbi_connect; + $self->{lock_depth} = 0; +} + +sub was_duplicate_error { + my $self = shift; + my $dbh = $self->dbh; + return 0 unless $dbh->err; + return 1 if $dbh->err == 1062 || $dbh->errstr =~ /duplicate/i; +} + +sub table_exists { + my ($self, $table) = @_; + return eval { + my $sth = $self->dbh->prepare("DESCRIBE $table"); + $sth->execute; + my $rec = $sth->fetchrow_hashref; + return $rec ? 1 : 0; + }; +} + +sub can_replace { 1 } +sub can_insertignore { 1 } +sub can_insert_multi { 1 } +sub unix_timestamp { "UNIX_TIMESTAMP()" } + +sub can_do_slaves { 0 } + +# Replication will be a plugin later +# sub check_slaves {} + +# return 1 on success. die otherwise. +sub clear_fsck_log { + my $self = shift; + $self->dbh->do("TRUNCATE TABLE fsck_log"); + return 1; +} + +# -------------------------------------------------------------------------- +# Functions specific to Store::Drizzle subclass. Not in parent. +# -------------------------------------------------------------------------- + +sub column_type { + my ($self, $table, $col) = @_; + my $sth = $self->dbh->prepare("DESCRIBE $table"); + $sth->execute; + while (my $rec = $sth->fetchrow_hashref) { + if ($rec->{Field} eq $col) { + $sth->finish; + return $rec->{Type}; + } + } + return undef; +} + +# -------------------------------------------------------------------------- +# Test suite things we override +# -------------------------------------------------------------------------- + +sub new_temp { + my $self = shift; + my %args = @_; + my $dbname = $args{dbname} || "tmp_mogiletest"; + my $host = $args{dbhost} || 'localhost'; + my $port = $args{dbport} || 3306; + my $user = $args{dbuser} || 'root'; + my $pass = $args{dbpass} || ''; + my $sto = + MogileFS::Store->new_from_dsn_user_pass("DBI:drizzle:database=$dbname;host=$host;port=$port", + $user, $pass); + + my $dbh = $sto->dbh; + _create_drizzle_db($dbh, $dbname); + + # allow MyISAM in the test suite. + $ENV{USE_UNSAFE_MYSQL} = 1 unless defined $ENV{USE_UNSAFE_MYSQL}; + + system("$FindBin::Bin/../mogdbsetup", "--yes", "--dbname=$dbname", + "--dbhost=$host", "--dbport=$port", "--dbrootuser=$user", + "--dbrootpass=$pass", "--dbuser=$user", "--dbpass=$pass") + and die "Failed to run mogdbsetup ($FindBin::Bin/../mogdbsetup)."; + + $dbh->do("use $dbname"); + return $sto; +} + +sub _create_drizzle_db { + my $dbh = shift; + my $dbname = shift; + _drop_drizzle_db($dbh, $dbname); + $dbh->do("CREATE DATABASE $dbname"); +} + +sub _drop_drizzle_db { + my $dbh = shift; + my $dbname = shift; + $dbh->do("DROP DATABASE IF EXISTS $dbname"); +} + +# -------------------------------------------------------------------------- +# Database creation time things we override +# -------------------------------------------------------------------------- + +sub create_table { + my $self = shift; + my ($table) = @_; + + my $dbh = $self->dbh; + my $errmsg = + "InnoDB backend is unavailable for use, force creation of tables " . + "by setting USE_UNSAFE_MYSQL=1 in your environment and run this " . + "command again."; + + unless ($ENV{USE_UNSAFE_MYSQL}) { + my $engines = eval { $dbh->selectall_hashref("SHOW ENGINES", "Engine"); }; + if ($@ && $dbh->err == 1064) { + # syntax error? for MySQL 4.0.x. + # who cares. we'll catch it below on the double-check. + } else { + die $errmsg + unless ($engines->{InnoDB} and + $engines->{InnoDB}->{Support} =~ m/^(YES|DEFAULT)$/i); + } + } + + my $existed = $self->table_exists($table); + + $self->SUPER::create_table(@_); + return if $ENV{USE_UNSAFE_MYSQL}; + + # don't alter an existing table up to InnoDB from MyISAM... + # could be costly. but on new tables, no problem... + unless ($existed) { + $dbh->do("ALTER TABLE $table ENGINE=InnoDB"); + warn "DBI reported an error of: '" . $dbh->errstr . "' when trying to " . + "alter table type of $table to InnoDB\n" if $dbh->err; + } + + # but in any case, let's see if it's already InnoDB or not. + my $table_status = $dbh->selectrow_hashref("SHOW TABLE STATUS LIKE '$table'"); + + # if not, either die or warn. + unless (($table_status->{Engine} || $table_status->{Type} || "") eq "InnoDB") { + if ($existed) { + warn "WARNING: Drizzle table that isn't InnoDB: $table\n"; + } else { + die "Drizzle didn't change table type to InnoDB as requested.\n\n$errmsg" + } + } + +} + +# -------------------------------------------------------------------------- +# Data-access things we override +# -------------------------------------------------------------------------- + +# update the device count for a given fidid +sub update_devcount_atomic { + my ($self, $fidid) = @_; + my $dbh = $self->dbh; + + my $saveAutoCommit = $dbh->{AutoCommit}; + $dbh->{AutoCommit}=0; + + my $result = $dbh->selectrow_array("SELECT fidid FROM file WHERE fidid = $fidid LOCK IN SHARE MODE"); + + if ( !$result ) { + $dbh->rollback; + } + + $self->update_devcount($fidid); + + $dbh->commit; + + $dbh->{AutoCommit}=$saveAutoCommit; + + return 1; +} + +sub should_begin_replicating_fidid { + my ($self, $fidid) = @_; + my $lockname = "mgfs:fid:$fidid:replicate"; + return 1 if $self->get_lock($lockname, 1); + return 0; +} + +sub note_done_replicating { + my ($self, $fidid) = @_; + my $lockname = "mgfs:fid:$fidid:replicate"; + $self->release_lock($lockname); +} + +sub upgrade_add_host_getport { + my $self = shift; + # see if they have the get port, else update it + unless ($self->column_type("host", "http_get_port")) { + $self->dowell("ALTER TABLE host ADD COLUMN http_get_port INT AFTER http_port"); + } + +} +sub upgrade_add_host_altip { + my $self = shift; + unless ($self->column_type("host", "altip")) { + $self->dowell("ALTER TABLE host ADD COLUMN altip VARBINARY(15) AFTER hostip"); + $self->dowell("ALTER TABLE host ADD COLUMN altmask VARBINARY(18) AFTER altip"); + $self->dowell("ALTER TABLE host ADD UNIQUE altip (altip)"); + } +} + +sub upgrade_add_device_asof { + my $self = shift; + unless ($self->column_type("device", "mb_asof")) { + $self->dowell("ALTER TABLE device ADD COLUMN mb_asof INT AFTER mb_used"); + } +} + +sub upgrade_add_device_weight { + my $self = shift; + unless ($self->column_type("device", "weight")) { + $self->dowell("ALTER TABLE device ADD COLUMN weight INT DEFAULT 100 AFTER status"); + } + +} + +sub upgrade_add_device_readonly { + my $self = shift; + unless ($self->column_type("device", "status") =~ /readonly/) { + $self->dowell("ALTER TABLE device MODIFY COLUMN status ENUM('alive', 'dead', 'down', 'readonly')"); + } +} + +sub upgrade_add_device_drain { + my $self = shift; + unless ($self->column_type("device", "status") =~ /drain/) { + $self->dowell("ALTER TABLE device MODIFY COLUMN status ENUM('alive', 'dead', 'down', 'readonly', 'drain')"); + } +} + +# get all hosts from database, returns them as list of hashrefs, hashrefs being the row contents. +sub get_all_hosts { + my ($self) = @_; + my $sth = $self->dbh->prepare("SELECT hostid, status, hostname, " . + "hostip, http_port, http_get_port, altip, altmask FROM host"); + $sth->execute; + my @ret; + while (my $row = $sth->fetchrow_hashref) { + push @ret, $row; + } + return @ret; +} + +# get all devices from database, returns them as list of hashrefs, hashrefs being the row contents. +sub get_all_devices { + my ($self) = @_; + my $sth = $self->dbh->prepare("SELECT devid, hostid, mb_total, " . + "mb_used, mb_asof, status, weight FROM device"); + $self->condthrow; + $sth->execute; + my @return; + while (my $row = $sth->fetchrow_hashref) { + push @return, $row; + } + return @return; +} + +sub TABLE_domain { + # keys can only be "767 bytes" in Drizzle, and VARCHAR is always utf8, so + # the namespace field length must be <= 191 instead of 255 + # (We could make it varbinary but people may have existing utf8 chars + # in their domains) + "CREATE TABLE domain ( + dmid INT NOT NULL PRIMARY KEY, + namespace VARCHAR(190), + UNIQUE (namespace) + )" +} + +sub TABLE_class { + "CREATE TABLE class ( + dmid INT NOT NULL, + classid INT NOT NULL, + PRIMARY KEY (dmid, classid), + classname VARCHAR(50), + UNIQUE (dmid,classname), + mindevcount INT NOT NULL + )" +} + +sub TABLE_file { + "CREATE TABLE file ( + fid INT NOT NULL, + PRIMARY KEY (fid), + + dmid INT NOT NULL, + dkey VARCHAR(190), # domain-defined + UNIQUE dkey (dmid, dkey), + + length INT, + + classid INT NOT NULL, + devcount INT NOT NULL, + INDEX devcount (dmid,classid,devcount) + )" +} + +sub TABLE_tempfile { + "CREATE TABLE tempfile ( + fid INT NOT NULL AUTO_INCREMENT, + PRIMARY KEY (fid), + + createtime INT NOT NULL, + classid INT NOT NULL, + dmid INT NOT NULL, + dkey VARCHAR(190), + devids VARCHAR(60) + )" +} + +sub TABLE_file_to_delete { + "CREATE TABLE file_to_delete ( + fid INT NOT NULL, + PRIMARY KEY (fid) + )" +} + +sub TABLE_unreachable_fids { + "CREATE TABLE unreachable_fids ( + fid INT NOT NULL, + lastupdate INT NOT NULL, + PRIMARY KEY (fid), + INDEX (lastupdate) + )" +} +sub TABLE_file_on { + "CREATE TABLE file_on ( + fid INT NOT NULL, + devid INT NOT NULL, + PRIMARY KEY (fid, devid), + INDEX (devid) + )" +} + +sub TABLE_file_on_corrupt { + "CREATE TABLE file_on_corrupt ( + fid INT NOT NULL, + devid INT NOT NULL, + PRIMARY KEY (fid, devid) + )" +} +sub TABLE_host { + "CREATE TABLE host ( + hostid INT NOT NULL PRIMARY KEY, + + status ENUM('alive','dead','down'), + http_port INT DEFAULT 7500, + http_get_port INT, + + hostname VARBINARY(40), + hostip VARBINARY(15), + altip VARBINARY(15), + altmask VARBINARY(18), + UNIQUE (hostname), + UNIQUE (hostip), + UNIQUE (altip) + )" +} + +# disks... +sub TABLE_device { + "CREATE TABLE device ( + devid INT NOT NULL, + hostid INT NOT NULL, + + status ENUM('alive','dead','down'), + weight INT DEFAULT 100, + + mb_total INT, + mb_used INT, + mb_asof INT, + PRIMARY KEY (devid), + INDEX (status) + )" +} +sub TABLE_server_settings { + "CREATE TABLE server_settings ( + field VARBINARY(50) PRIMARY KEY, + value VARCHAR(255) + )" +} + +sub TABLE_file_to_replicate { + "CREATE TABLE file_to_replicate ( + fid INT NOT NULL PRIMARY KEY, + nexttry INT NOT NULL, + INDEX (nexttry), + fromdevid INT , + failcount INT NOT NULL DEFAULT 0, + flags INT NOT NULL DEFAULT 0 + )" +} + +sub TABLE_file_to_delete_later { + "CREATE TABLE file_to_delete_later ( + fid INT NOT NULL PRIMARY KEY, + delafter INT NOT NULL, + INDEX (delafter) + )" +} +sub TABLE_fsck_log { + "CREATE TABLE fsck_log ( + logid INT NOT NULL AUTO_INCREMENT, + PRIMARY KEY (logid), + utime INT NOT NULL, + fid INT NULL, + evcode CHAR(4), + devid INT, + INDEX(utime) + )" +} + +sub TABLE_file_to_queue { + "CREATE TABLE file_to_queue ( + fid INT NOT NULL, + devid INT, + type INT NOT NULL, + nexttry INT NOT NULL, + failcount INT NOT NULL default '0', + flags INT NOT NULL default '0', + PRIMARY KEY (fid, type), + INDEX type_nexttry (type,nexttry) + )" +} +sub TABLE_file_to_delete2 { + "CREATE TABLE file_to_delete2 ( + fid INT NOT NULL PRIMARY KEY, + nexttry INT NOT NULL, + failcount INT NOT NULL default '0', + INDEX nexttry (nexttry) + )" +} + + + +1; + +__END__ + +=head1 NAME + +MogileFS::Store::Drizzle - Drizzle data storage for MogileFS + +=head1 SEE ALSO + +L + + diff -urN mogilefs-server-2.30.orig/lib/MogileFS/Store.pm mogilefs-server-2.30/lib/MogileFS/Store.pm --- mogilefs-server-2.30.orig/lib/MogileFS/Store.pm 2009-01-12 03:19:57.000000000 -0800 +++ mogilefs-server-2.30/lib/MogileFS/Store.pm 2009-02-24 10:33:55.000000000 -0800 @@ -26,6 +26,8 @@ my $subclass; if ($dsn =~ /^DBI:mysql:/i) { $subclass = "MogileFS::Store::MySQL"; + } elsif ($dsn =~ /^DBI:drizzle:/i) { + $subclass = "MogileFS::Store::Drizzle"; } elsif ($dsn =~ /^DBI:SQLite:/i) { $subclass = "MogileFS::Store::SQLite"; } elsif ($dsn =~ /^DBI:Oracle:/i) {