Skip to content

Commit

Permalink
Merge pull request #1 from alibaba/master
Browse files Browse the repository at this point in the history
pull
  • Loading branch information
rewerma committed Jun 11, 2018
2 parents 4141049 + 77f1dfd commit ad20166
Show file tree
Hide file tree
Showing 31 changed files with 5,237 additions and 4,312 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -457,8 +457,10 @@ private void waitClientRunning() {
}

running = true;

mutex.get();// 阻塞等待
} else {
// 单机模式直接设置为running
running = true;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ static void putEntry(final int charsetId, String mysqlCharset, String mysqlColla
putEntry(42, "latin7", "latin7_general_cs", "ISO8859_7");
putEntry(43, "macce", "macce_bin", "MacCentralEurope");
putEntry(44, "cp1250", "cp1250_croatian_ci", "Cp1250");
putEntry(45, "utf8mb4", "utf8mb4_general_ci", "MacCentralEurope");
putEntry(46, "utf8mb4", "utf8mb4_bin", "MacCentralEurope");
putEntry(45, "utf8mb4", "utf8mb4_general_ci", "UTF-8");
putEntry(46, "utf8mb4", "utf8mb4_bin", "UTF-8");
putEntry(47, "latin1", "latin1_bin", "ISO8859_1");
putEntry(48, "latin1", "latin1_general_ci", "ISO8859_1");
putEntry(49, "latin1", "latin1_general_cs", "ISO8859_1");
Expand Down
35 changes: 32 additions & 3 deletions dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,19 @@
import com.taobao.tddl.dbsync.binlog.event.StartLogEventV3;
import com.taobao.tddl.dbsync.binlog.event.StopLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TransactionContextLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.ViewChangeEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XaPrepareLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.StartEncryptionLogEvent;

/**
* Implements a binary-log decoder.
Expand Down Expand Up @@ -366,6 +370,24 @@ public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext con
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.TRANSACTION_CONTEXT_EVENT: {
TransactionContextLogEvent event = new TransactionContextLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.VIEW_CHANGE_EVENT: {
ViewChangeEvent event = new ViewChangeEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.XA_PREPARE_LOG_EVENT: {
XaPrepareLogEvent event = new XaPrepareLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.ANNOTATE_ROWS_EVENT: {
AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
/* updating position in context */
Expand All @@ -390,6 +412,12 @@ public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext con
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.START_ENCRYPTION_EVENT: {
StartEncryptionLogEvent event = new StartEncryptionLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
default:
/*
* Create an object of Ignorable_log_event for unrecognized
Expand All @@ -402,9 +430,10 @@ public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext con
logPosition.position = header.getLogPos();
return event;
} else {
if (logger.isWarnEnabled()) logger.warn("Skipping unrecognized binlog event "
+ LogEvent.getTypeName(header.getType()) + " from: "
+ context.getLogPosition());
if (logger.isWarnEnabled()) {
logger.warn("Skipping unrecognized binlog event " + LogEvent.getTypeName(header.getType())
+ " from: " + context.getLogPosition());
}
}
}

Expand Down
30 changes: 19 additions & 11 deletions dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,17 @@ public abstract class LogEvent {

public static final int PREVIOUS_GTIDS_LOG_EVENT = 35;

/* MySQL 5.7 events */
public static final int TRANSACTION_CONTEXT_EVENT = 36;

public static final int VIEW_CHANGE_EVENT = 37;

/* Prepared XA transaction terminal event similar to Xid */
public static final int XA_PREPARE_LOG_EVENT = 38;

// mariaDb 5.5.34
/* New MySQL/Sun events are to be added right above this comment */
public static final int MYSQL_EVENTS_END = 36;
public static final int MYSQL_EVENTS_END = 39;

public static final int MARIA_EVENTS_BEGIN = 160;
/* New Maria event numbers start from here */
Expand All @@ -189,8 +197,10 @@ public abstract class LogEvent {
*/
public static final int GTID_LIST_EVENT = 163;

public static final int START_ENCRYPTION_EVENT = 164;

/** end marker */
public static final int ENUM_END_EVENT = 164;
public static final int ENUM_END_EVENT = 165;

/**
* 1 byte length, 1 byte format Length is total length in bytes, including 2
Expand Down Expand Up @@ -359,7 +369,7 @@ public static String getTypeName(final int type) {
protected static final Log logger = LogFactory.getLog(LogEvent.class);

protected final LogHeader header;

/**
* mysql半同步semi标识
*
Expand All @@ -369,18 +379,16 @@ public static String getTypeName(final int type) {
* </pre>
*/
protected int semival;



public int getSemival() {
return semival;
}
return semival;
}

public void setSemival(int semival) {
this.semival = semival;
}
public void setSemival(int semival) {
this.semival = semival;
}

protected LogEvent(LogHeader header){
protected LogEvent(LogHeader header){
this.header = header;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
public static final int HEARTBEAT_HEADER_LEN = 0;
public static final int IGNORABLE_HEADER_LEN = 0;
public static final int ROWS_HEADER_LEN_V2 = 10;
public static final int TRANSACTION_CONTEXT_HEADER_LEN = 18;
public static final int VIEW_CHANGE_HEADER_LEN = 52;
public static final int XA_PREPARE_HEADER_LEN = 0;

public static final int ANNOTATE_ROWS_HEADER_LEN = 0;
public static final int BINLOG_CHECKPOINT_HEADER_LEN = 4;
public static final int GTID_HEADER_LEN = 19;
public static final int GTID_LIST_HEADER_LEN = 4;
public static final int START_ENCRYPTION_HEADER_LEN = 0;

public static final int POST_HEADER_LENGTH = 11;

Expand Down Expand Up @@ -202,11 +207,17 @@ public FormatDescriptionLogEvent(final int binlogVersion){
postHeaderLen[GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
postHeaderLen[ANONYMOUS_GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
postHeaderLen[PREVIOUS_GTIDS_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;

postHeaderLen[TRANSACTION_CONTEXT_EVENT - 1] = TRANSACTION_CONTEXT_HEADER_LEN;
postHeaderLen[VIEW_CHANGE_EVENT - 1] = VIEW_CHANGE_HEADER_LEN;
postHeaderLen[XA_PREPARE_LOG_EVENT - 1] = XA_PREPARE_HEADER_LEN;

// mariadb 10
postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;
postHeaderLen[BINLOG_CHECKPOINT_EVENT - 1] = BINLOG_CHECKPOINT_HEADER_LEN;
postHeaderLen[GTID_EVENT - 1] = GTID_HEADER_LEN;
postHeaderLen[GTID_LIST_EVENT - 1] = GTID_LIST_HEADER_LEN;
postHeaderLen[START_ENCRYPTION_EVENT - 1] = START_ENCRYPTION_HEADER_LEN;
break;

case 3: /* 4.0.x x>=2 */
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package com.taobao.tddl.dbsync.binlog.event;

import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;

import java.nio.ByteBuffer;
import java.util.UUID;

import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;

/**
* @author jianghang 2013-4-8 上午12:36:29
* @version 1.0.3
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package com.taobao.tddl.dbsync.binlog.event;

import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;

/**
* @author agapple 2018年5月7日 下午7:05:39
* @version 1.0.26
* @since mysql 5.7
*/
public class TransactionContextLogEvent extends LogEvent {

public TransactionContextLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package com.taobao.tddl.dbsync.binlog.event;

import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;

/**
* @author agapple 2018年5月7日 下午7:05:39
* @version 1.0.26
* @since mysql 5.7
*/
public class ViewChangeEvent extends LogEvent {

public ViewChangeEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package com.taobao.tddl.dbsync.binlog.event;

import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;

/**
* @author agapple 2018年5月7日 下午7:05:39
* @version 1.0.26
* @since mysql 5.7
*/
public class XaPrepareLogEvent extends LogEvent {

private boolean onePhase;
private int formatId;
private int gtridLength;
private int bqualLength;
private byte[] data;

public XaPrepareLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);

final int commonHeaderLen = descriptionEvent.getCommonHeaderLen();
final int postHeaderLen = descriptionEvent.getPostHeaderLen()[header.getType() - 1];

int offset = commonHeaderLen + postHeaderLen;
buffer.position(offset);

onePhase = (buffer.getInt8() == 0x00 ? false : true);

formatId = buffer.getInt32();
gtridLength = buffer.getInt32();
bqualLength = buffer.getInt32();

int MY_XIDDATASIZE = 128;
if (MY_XIDDATASIZE >= gtridLength + bqualLength && gtridLength >= 0 && gtridLength <= 64 && bqualLength >= 0
&& bqualLength <= 64) {
data = buffer.getData(gtridLength + bqualLength);
} else {
formatId = -1;
gtridLength = 0;
bqualLength = 0;
}
}

public boolean isOnePhase() {
return onePhase;
}

public int getFormatId() {
return formatId;
}

public int getGtridLength() {
return gtridLength;
}

public int getBqualLength() {
return bqualLength;
}

public byte[] getData() {
return data;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.taobao.tddl.dbsync.binlog.event.mariadb;

import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
import com.taobao.tddl.dbsync.binlog.event.LogHeader;

/**
* mariadb的Start_encryption_log_event
*
* @author agapple 2018年5月7日 下午7:23:02
* @version 1.0.26
*/
public class StartEncryptionLogEvent extends LogEvent {

public StartEncryptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
}
}
5 changes: 3 additions & 2 deletions deployer/src/main/resources/example/instance.properties
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
#################################################
## mysql serverId
canal.instance.mysql.slaveId=0

# position info
canal.instance.master.address=127.0.0.1:3306
# enable gtid use true/false
canal.instance.gtidon=false
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=

canal.instance.gtidon=true

# table meta tsdb info
canal.instance.tsdb.enable=true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
Expand Down
5 changes: 5 additions & 0 deletions deployer/src/main/resources/spring/default-instance.xml
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,15 @@
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
Expand All @@ -187,5 +189,8 @@
<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>
</bean>
</beans>
2 changes: 1 addition & 1 deletion deployer/src/main/resources/spring/file-instance.xml
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,6 @@
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon}"/>
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>
</bean>
</beans>
Loading

0 comments on commit ad20166

Please sign in to comment.