小米信息部技术团队2021-05-24T03:39:30.206Zhttps://xiaomi-info.github.io/小米信息部技术团队Hexo统计建模初探 —— Analysis of Correlationhttps://xiaomi-info.github.io/2020/05/24/analysis-of-correlation/2020-05-24T09:15:00.000Z2021-05-24T03:39:30.206Z统计建模初探 —— Analysis of Correlation
import math import random import numpy as np # 计算期望 defmean(x): return sum(x)/len(x) # 计算每一项数据与均值的差 defde_mean(x): x_bar = mean(x) return [x_i - x_bar for x_i in x] defdot(v, w): return sum(v_i * w_i for v_i, w_i in zip(v, w)) defsum_of_squares(v): return dot(v, v) # 方差 defvariance(x): n = len(x) deviations = de_mean(x) return sum_of_squares(deviations) / (n - 1) # 标准差 defstandard_deviation(x): return math.sqrt(variance(x)) # 协方差 defcovariance(x, y): n = len(x) return dot(de_mean(x), de_mean(y)) / (n - 1)
# 随机生成数据 a = [random.randint(0, 100) for i in range(25)] b = [random.randint(0, 100) for j in range(25)] print(a) print(b) print(covariance(a, b)) # 构造矩阵 ab = np.array([a, b]) # 计算协方差矩阵 print(np.cov(ab))
a = [random.randint(0, 100) for i in range(20)] b = [random.randint(0, 100) for j in range(20)] print(a) print(b) print(correlation(a, b)) # 构造矩阵 ab = np.array([a, b]) # 计算相关系数 print(np.corrcoef(ab))
At runtime and in the builder environment we need to be able to figure out which properties, events, and methods a Java Bean supports. We call this process introspection.
By default we will use a low level reflection mechanism to study the methods supported by a target bean and then apply simple design patterns to deduce from those methods what properties, events, and public methods are supported.
Java Beans 内省机制
核心类库
Java Beans 内省机制的核心类是 Introspector：
1 2
* The Introspector classprovidesastandardwayfortoolstolearnabout * theproperties, events, andmethodssupportedbyatargetJavaBean.
@Test publicvoidtest3(){ User user = new User(); user.setAge(1); user.setUsername("zhangsan"); user.addPropertyChangeListener(System.out::println); user.setUsername("lisi"); user.setUsername("wangwu"); }
at introspector.test.IntrospectorTest.lambda$test3$1(IntrospectorTest.java:78) at java.beans.VetoableChangeSupport.fireVetoableChange(VetoableChangeSupport.java:375)
final ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("storm-client.xml"); final HelloService helloService = (HelloService) context.getBean("helloService"); String result = helloService.sayHello("World"); System.out.println(result); for (;;) {
// The implementation of the predeclared function panic. funcgopanic(e interface{}) { gp := getg() if gp.m.curg != gp { print("panic: ") printany(e) print("\n") throw("panic on system stack") }
if gp.m.mallocing != 0 { print("panic: ") printany(e) print("\n") throw("panic during malloc") } if gp.m.preemptoff != "" { print("panic: ") printany(e) print("\n") print("preempt off reason: ") print(gp.m.preemptoff) print("\n") throw("panic during preemptoff") } if gp.m.locks != 0 { print("panic: ") printany(e) print("\n") throw("panic holding locks") }
var p _panic p.arg = e p.link = gp._panic gp._panic = (*_panic)(noescape(unsafe.Pointer(&p)))
atomic.Xadd(&runningPanicDefers, 1)
for { d := gp._defer if d == nil { break }
// If defer was started by earlier panic or Goexit (and, since we're back here, that triggered a new panic), // take defer off list. The earlier panic or Goexit will not continue running. if d.started { if d._panic != nil { d._panic.aborted = true } d._panic = nil d.fn = nil gp._defer = d.link freedefer(d) continue }
// Mark defer as started, but keep on list, so that traceback // can find and update the defer's argument frame if stack growth // or a garbage collection happens before reflectcall starts executing d.fn. d.started = true
// Record the panic that is running the defer. // If there is a new panic during the deferred call, that panic // will find d in the list and will mark d._panic (this panic) aborted. d._panic = (*_panic)(noescape(unsafe.Pointer(&p)))
// reflectcall did not panic. Remove d. if gp._defer != d { throw("bad defer entry in panic") } d._panic = nil d.fn = nil gp._defer = d.link
// trigger shrinkage to test stack copy. See stack_test.go:TestStackPanic //GC()
pc := d.pc sp := unsafe.Pointer(d.sp) // must be pointer so it gets adjusted during stack copy freedefer(d) if p.recovered { atomic.Xadd(&runningPanicDefers, -1)
gp._panic = p.link // Aborted panics are marked but remain on the g.panic list. // Remove them from the list. for gp._panic != nil && gp._panic.aborted { gp._panic = gp._panic.link } if gp._panic == nil { // must be done with signal gp.sig = 0 } // Pass information about recovering frame to recovery. gp.sigcode0 = uintptr(sp) gp.sigcode1 = pc mcall(recovery) throw("recovery failed") // mcall should not return } }
// ran out of deferred calls - old-school panic now // Because it is unsafe to call arbitrary user code after freezing // the world, we call preprintpanics to invoke all necessary Error // and String methods to prepare the panic strings before startpanic. preprintpanics(gp._panic)
fatalpanic(gp._panic) // should not return *(*int)(nil) = 0// not reached }
// The implementation of the predeclared function recover. // Cannot split the stack because it needs to reliably // find the stack segment of its caller. // // TODO(rsc): Once we commit to CopyStackAlways, // this doesn't need to be nosplit. //go:nosplit funcgorecover(argp uintptr)interface{} { // Must be in a function running as part of a deferred call during the panic. // Must be called from the topmost function of the call // (the function used in the defer statement). // p.argp is the argument pointer of that topmost deferred function call. // Compare against argp reported by caller. // If they match, the caller is the one who can recover. gp := getg() p := gp._panic if p != nil && !p.recovered && argp == uintptr(p.argp) { p.recovered = true return p.arg } returnnil }
// func mcall(fn func(*g)) // Switch to m->g0's stack, call fn(g). // Fn must never return. It should gogo(&g->sched) // to keep running g. TEXT runtime·mcall(SB), NOSPLIT, $0-8 MOVQfn+0(FP), DI
get_tls(CX) MOVQg(CX), AX// save state in g->sched MOVQ0(SP), BX// caller's PC MOVQBX, (g_sched+gobuf_pc)(AX) LEAQfn+0(FP), BX// caller's SP MOVQBX, (g_sched+gobuf_sp)(AX) MOVQAX, (g_sched+gobuf_g)(AX) MOVQBP, (g_sched+gobuf_bp)(AX)
// switch to m->g0 & its stack, call fn MOVQg(CX), BX MOVQg_m(BX), BX MOVQm_g0(BX), SI CMPQSI, AX// if g == m->g0 call badmcall JNE3(PC) MOVQ$runtime·badmcall(SB), AX JMPAX MOVQSI, g(CX)// g = m->g0 MOVQ(g_sched+gobuf_sp)(SI), SP// sp = m->g0->sched.sp PUSHQAX MOVQDI, DX MOVQ0(DI), DI CALLDI POPQAX MOVQ$runtime·badmcall2(SB), AX JMPAX RET
// Unwind the stack after a deferred function calls recover // after a panic. Then arrange to continue running as though // the caller of the deferred function returned normally. funcrecovery(gp *g) { // Info about defer passed in G struct. sp := gp.sigcode0 pc := gp.sigcode1
// d's arguments need to be in the stack. if sp != 0 && (sp < gp.stack.lo || gp.stack.hi < sp) { print("recover: ", hex(sp), " not in [", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n") throw("bad recovery") }
// Make the deferproc for this d return again, // this time returning 1. The calling function will // jump to the standard return epilogue. gp.sched.sp = sp gp.sched.pc = pc gp.sched.lr = 0 gp.sched.ret = 1 gogo(&gp.sched) }
/* * generic single channel send/recv * If block is not nil, * then the protocol will not * sleep but return if it could * not complete. * * sleep can wake up with g.param == nil * when a channel involved in the sleep has * been closed. it is easiest to loop and re-run * the operation; we'll see that it's now closed. */ funcchansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr)bool { if c == nil { if !block { returnfalse } gopark(nil, nil, waitReasonChanSendNilChan, traceEvGoStop, 2) throw("unreachable") }
if debugChan { print("chansend: chan=", c, "\n") }
if raceenabled { racereadpc(c.raceaddr(), callerpc, funcPC(chansend)) }
// Fast path: check for failed non-blocking operation without acquiring the lock. // // After observing that the channel is not closed, we observe that the channel is // not ready for sending. Each of these observations is a single word-sized read // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). // Because a closed channel cannot transition from 'ready for sending' to // 'not ready for sending', even if the channel is closed between the two observations, // they imply a moment between the two when the channel was both not yet closed // and not ready for sending. We behave as if we observed the channel at that moment, // and report that the send cannot proceed. // // It is okay if the reads are reordered here: if we observe that the channel is not // ready for sending and then observe that it is not closed, that implies that the // channel wasn't closed during the first observation. if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { returnfalse }
var t0 int64 if blockprofilerate > 0 { t0 = cputicks() }
lock(&c.lock)
if c.closed != 0 { unlock(&c.lock) panic(plainError("send on closed channel")) }
if sg := c.recvq.dequeue(); sg != nil { // Found a waiting receiver. We pass the value we want to send // directly to the receiver, bypassing the channel buffer (if any). send(c, sg, ep, func() { unlock(&c.lock) }, 3) returntrue }
if c.qcount < c.dataqsiz { // Space is available in the channel buffer. Enqueue the element to send. qp := chanbuf(c, c.sendx) if raceenabled { raceacquire(qp) racerelease(qp) } typedmemmove(c.elemtype, qp, ep) c.sendx++ if c.sendx == c.dataqsiz { c.sendx = 0 } c.qcount++ unlock(&c.lock) returntrue }
if !block { unlock(&c.lock) returnfalse }
// Block on the channel. Some receiver will complete our operation for us. gp := getg() mysg := acquireSudog() mysg.releasetime = 0 if t0 != 0 { mysg.releasetime = -1 } // No stack splits between assigning elem and enqueuing mysg // on gp.waiting where copystack can find it. mysg.elem = ep mysg.waitlink = nil mysg.g = gp mysg.isSelect = false mysg.c = c gp.waiting = mysg gp.param = nil c.sendq.enqueue(mysg) goparkunlock(&c.lock, waitReasonChanSend, traceEvGoBlockSend, 3) // Ensure the value being sent is kept alive until the // receiver copies it out. The sudog has a pointer to the // stack object, but sudogs aren't considered as roots of the // stack tracer. KeepAlive(ep)
// someone woke us up. if mysg != gp.waiting { throw("G waiting list is corrupted") } gp.waiting = nil if gp.param == nil { if c.closed == 0 { throw("chansend: spurious wakeup") } panic(plainError("send on closed channel")) } gp.param = nil if mysg.releasetime > 0 { blockevent(mysg.releasetime-t0, 2) } mysg.c = nil releaseSudog(mysg) returntrue }
funcfoo(){ deferfunc(){ if err := recover(); err != nil { fmt.Println(err) } }() var i interface{} = "abc" _ = i.([]string) }
funcmain(){ foo() fmt.Println("exit") }
输出：
1 2
interface conversion: interface {} is string, not []string exit
源码在 src/runtime/iface.go 中，如下两个函数：
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
// panicdottypeE is called when doing an e.(T) conversion and the conversion fails. // have = the dynamic type we have. // want = the static type we're trying to convert to. // iface = the static type we're converting from. funcpanicdottypeE(have, want, iface *_type) { panic(&TypeAssertionError{iface, have, want, ""}) }
// panicdottypeI is called when doing an i.(T) conversion and the conversion fails. // Same args as panicdottypeE, but "have" is the dynamic itab we have. funcpanicdottypeI(have *itab, want, iface *_type) { var t *_type if have != nil { t = have._type } panicdottypeE(t, want, iface) }
分布式事务是指事务的参与者、支持事务的服务器、资源服务器以及事务管理器分别位于不同的分布式系统的不同节点之上。例如在大型电商系统中，下单接口通常会扣减库存、减去优惠、生成订单 id, 而订单服务与库存、优惠、订单 id 都是不同的服务，下单接口的成功与否，不仅取决于本地的 db 操作，而且依赖第三方系统的结果，这时候分布式事务就保证这些操作要么全部成功，要么全部失败。本质上来说，分布式事务就是为了保证不同数据库的数据一致性。
CAP 原则又称 CAP 定理，指的是在一个分布式系统中， Consistency（一致性）、 Availability（可用性）、Partition tolerance（分区容错性），三者不可得兼。
一致性（C）：
在分布式系统中的所有数据备份，在同一时刻是否同样的值。（等同于所有节点访问同一份最新的数据副本）
可用性（A）：
在集群中一部分节点故障后，集群整体是否还能响应客户端的读写请求。（对数据更新具备高可用性）
分区容错性（P）：
以实际效果而言，分区相当于对通信的时限要求。系统如果不能在时限内达成数据一致性，就意味着发生了分区的情况，必须就当前操作在 C 和 A 之间做出选择。
CAP 原则的精髓就是要么 AP，要么 CP，要么 AC，但是不存在 CAP。如果在某个分布式系统中数据无副本， 那么系统必然满足强一致性条件， 因为只有独一数据，不会出现数据不一致的情况，此时 C 和 P 两要素具备，但是如果系统发生了网络分区状况或者宕机，必然导致某些数据不可以访问，此时可用性条件就不能被满足，即在此情况下获得了 CP 系统，但是 CAP 不可同时满足。
BASE 理论
BASE 理论指的是基本可用 Basically Available，软状态 Soft State，最终一致性 Eventual Consistency，核心思想是即便无法做到强一致性，但应该采用适合的方式保证最终一致性。
BASE，Basically Available Soft State Eventual Consistency 的简写： BA：Basically Available 基本可用，分布式系统在出现故障的时候，允许损失部分可用性，即保证核心可用。 S：Soft State 软状态，允许系统存在中间状态，而该中间状态不会影响系统整体可用性。 E：Consistency 最终一致性，系统中的所有数据副本经过一定时间后，最终能够达到一致的状态。 BASE 理论本质上是对 CAP 理论的延伸，是对 CAP 中 AP 方案的一个补充。
柔性事务
不同于 ACID 的刚性事务，在分布式场景下基于 BASE 理论，就出现了柔性事务的概念。要想通过柔性事务来达到最终的一致性，就需要依赖于一些特性，这些特性在具体的方案中不一定都要满足，因为不同的方案要求不一样；但是都不满足的话，是不可能做柔性事务的。
转账是最经典那的分布式事务场景，假设用户 A 使用银行 app 发起一笔跨行转账给用户 B，银行系统首先扣掉用户 A 的钱，然后增加用户 B 账户中的余额。此时就会出现 2 种异常情况：1. 用户 A 的账户扣款成功，用户 B 账户余额增加失败 2. 用户 A 账户扣款失败，用户 B 账户余额增加成功。对于银行系统来说，以上 2 种情况都是不允许发生，此时就需要分布式事务来保证转账操作的成功。
下单扣库存
在电商系统中，下单是用户最常见操作。在下单接口中必定会涉及生成订单 id, 扣减库存等操作，对于微服务架构系统，订单 id 与库存服务一般都是独立的服务，此时就需要分布式事务来保证整个下单接口的成功。
跨行转账可通过该方案实现。 用户 A 向用户 B 发起转账，首先系统会扣掉用户 A 账户中的金额，将该转账消息写入消息表中，如果事务执行失败则转账失败，如果转账成功，系统中会有定时轮询消息表，往 mq 中写入转账消息，失败重试。mq 消息会被实时消费并往用户 B 中账户增加转账金额，执行失败会不断重试。
If one or both arguments are NULL, the result of the comparison is NULL, except for the NULL-safe <=> equality comparison operator. For NULL <=> NULL, the result is true. No conversion is needed.
If both arguments in a comparison operation are strings, they are compared as strings.
If both arguments are integers, they are compared as integers.
Hexadecimal values are treated as binary strings if not compared to a number.
If one of the arguments is a TIMESTAMP or DATETIME column and the other argument is a constant, the constant is converted to a timestamp before the comparison is performed. This is done to be more ODBC-friendly. This is not done for the arguments to IN(). To be safe, always use complete datetime, date, or time strings when doing comparisons. For example, to achieve best results when using BETWEEN with date or time values, use CAST() to explicitly convert the values to the desired data type.
A single-row subquery from a table or tables is not considered a constant. For example, if a subquery returns an integer to be compared to a DATETIME value, the comparison is done as two integers. The integer is not converted to a temporal value. To compare the operands as DATETIME values, use CAST() to explicitly convert the subquery value to DATETIME.
If one of the arguments is a decimal value, comparison depends on the other argument. The arguments are compared as decimal values if the other argument is a decimal or integer value, or as floating-point values if the other argument is a floating-point value.
In all other cases, the arguments are compared as floating-point (real) numbers. For example, a comparison of string and numeric operands takes places as a comparison of floating-point numbers.
The server includes dtoa, a conversion library that provides the basis for improved conversion > between string or DECIMAL values and approximate-value (FLOAT/DOUBLE) numbers:
Consistent conversion results across platforms, which eliminates, for example, Unix versus > Windows conversion differences.
Accurate representation of values in cases where results previously did not provide sufficient > precision, such as for values close to IEEE limits.
Conversion of numbers to string format with the best possible precision. The precision of dtoa > is always the same or better than that of the standard C library functions.
MySQL 里使用了 dtoa 这个转换类库来实现字符跟浮点数的相互转换，一是提高了浮点数的精度（相比 c 标准库）, 二是保持平台一致性。
在 MySQL 里浮点数之间的比较是近似的。因为 MySQL 里浮点数是不精确的，它的精度位只有 53 比特，大于 53 比特的位被“四舍五入”。这会导致一些违反直观的结果。 比如下面两个 SQL
insertinto t3 values(1234567890123456789); insertinto t3 values(123456789012345678); insertinto t3 values(123456789012345677); insertinto t3 values(12345678901234567); insertinto t3 values(12345678901234568); insertinto t3 values(123456789012345);
select * from t3 where c1='123456789012345678'; +--------------------+ | c1 | +--------------------+ | 123456789012345678 | +--------------------+
此处结果跟前面的规则貌似不一致？ 在 MySQL 执行的 prepare 阶段，在设置比较方法之前，满足一些调教的情况下会做一次类型转换
/** Convert a constant item to an int and replace the original item. The function converts a constant expression or string to an integer. On successful conversion the original item is substituted for the result of the item evaluation. This is done when comparing DATE/TIME of different formats and also when comparing bigint to strings (in which case strings are converted to bigints). @param thd thread handle @param field item will be converted using the type of this field @param[in,out] item reference to the item to convert @note This function is called only at prepare stage. As all derived tables are filled only after all derived tables are prepared we do not evaluate items with subselects here because they can contain derived tables and thus we may attempt to use a table that has not been populated yet. @retval 0 Can't convert item @retval 1 Item was replaced with an integer version of the item */
for { clientConn, err := listener.Accept() if err != nil { if nerr, ok := err.(net.Error); ok && nerr.Temporary() { logf(lg.WARN, "temporary Accept() failure - %s", err) runtime.Gosched() continue } // theres no direct way to detect this error because it is not exposed if !strings.Contains(err.Error(), "use of closed network connection") { return fmt.Errorf("listener.Accept() error - %s", err) } break } go handler.Handle(clientConn) }
func(n *NSQD)GetTopic(topicName string) *Topic { // most likely, we already have this topic, so try read lock first. n.RLock() t, ok := n.topicMap[topicName] n.RUnlock() if ok { return t }
n.Lock()
t, ok = n.topicMap[topicName] if ok { n.Unlock() return t } deleteCallback := func(t *Topic) { n.DeleteExistingTopic(t.name) } t = NewTopic(topicName, &context{n}, deleteCallback) n.topicMap[topicName] = t
这里操作系统是一个软件，CPU、寄存器、内存（物理内存）都是实打实的硬件。操作系统虽然也是一堆代码写出来的。但是她是硬件对其它应用程序的接口。总的来讲操作系统通过系统调用控制所有的硬件资源，他把其它的程序调度到 CPU 上让其它程序执行，但是为了让每个程序都有机会使用 CPU，CPU 又通过时间中断把控制权交给操作系统。
最后我们的代码被编译成机器码之后，本质就是一条条的指令。我们期望的就是 CPU 去执行完这些指令进而完成任务。而操作系统又能够帮助我们让 CPU 来执行代码以及提供所需资源的调用接口（系统调用）。是不是非常简单？
Go 程序的调用规约
在上面我们知道整个虚拟内存被我们划分为：代码区、静态数据区、栈区、堆区。接下来要讲的 Go 程序的调用规约（其实就是函数、方法运行的规则），主要是涉及上面所说的栈部分（堆部分会在内存分配的文章里边去讲）。以及计算机软硬各个部分如何配合。接下来我们就来看一下程序的基本单位函数跟方法是怎么执行与相互调用的。
The AST is then type-checked. The first steps are name resolution and type inference, which determine which object belongs to which identifier, and what type each expression has. Type-checking includes certain extra checks, such as “declared and not used” as well as determining whether or not a function terminates.
Certain transformations are also done on the AST. Some nodes are refined based on type information, such as string additions being split from the arithmetic addition node type. Some other examples are dead code elimination, function call inlining, and escape analysis.
这一段是说：AST 也会进行转换，有些节点根据类型信息进行精简，比如从算术加法节点类型中拆分出字符串加法。其它一些例子像 dead code 的消除，函数调用内联和逃逸分析。
但是，我们有各种各样的操作系统，有不同的 CPU 类型，每一种的位数可能不同；寄存器能够使用的指令也不同，像是复杂指令集与精简指令集等；在进行各个平台的兼容之前，我们还需要替换一些底层函数，比如我们使用 make 来初始化 slice，此时会根据传入的类型替换为：makeslice64 或者 makeslice。当然还有像 painc、channel 等等函数的替换也会在中间码生成过程中进行替换。这一部分的替换操作可以在这里查看。
整个过程下来，可以看到编译器后端有很多工作需要做的，你需要对某一个指令集、cpu 的架构了解，才能正确的进行翻译机器码。同时不能仅仅是正确，一个语言的效率是高还是低，也在很大程度上取决于编译器后端的优化。特别是即将进入 AI 时代，越来越多的芯片厂商诞生，我估计以后对这方面人才的需求会变得越来越旺盛。
